diff --git a/.agents/skills/serialization-audit/SKILL.md b/.agents/skills/serialization-audit/SKILL.md new file mode 100644 index 0000000000..dd73c8b9bd --- /dev/null +++ b/.agents/skills/serialization-audit/SKILL.md @@ -0,0 +1,122 @@ +--- +name: serialization-audit +description: > + Use this skill when verifying serialization behavior across branches, testing for backwards + compatibility in JSON serialization changes, or comparing API request/response/storage formats + between implementations. Apply when migrating serializers (e.g., Newtonsoft to System.Text.Json), + adding new JSON converters, or changing naming policies. +--- + +# Serialization Audit Skill + +## Overview + +The serialization audit workflow generates branch-specific JSON snapshots of API behavior and compares them to detect behavioral differences. It exercises the full pipeline: API submission → queue → job processing → Elasticsearch storage → API response. + +## Workflow + +### 1. Run Audit Script on Both Branches + +The audit runner lives at `.agents/skills/serialization-audit/scripts/audit-api-surface.ps1`. It: +- Submit events with different JSON casing conventions (snake_case, PascalCase, camelCase, mixed) +- Capture the raw request body, Elasticsearch stored document, and API response +- Save each to `audit-output/{audit-run-id}/{branch-name}/{scenario}/` with files like: + - `request.json` — what was submitted + - `elastic.json` — what was stored + - `response.json` — what the API returned + +```bash +# Start Exceptionless locally first: +aspire run + +# On main branch: +git checkout main +pwsh .agents/skills/serialization-audit/scripts/audit-api-surface.ps1 \ + -BranchName main \ + -AuditRunId live-serialization + +# On feature branch: +git checkout feature/system-text-json-v2 +pwsh .agents/skills/serialization-audit/scripts/audit-api-surface.ps1 \ + -BranchName feature-system-text-json-v2 \ + -AuditRunId live-serialization +``` + +**Requirements:** API and Elasticsearch must be running locally through Aspire. + +### 2. Diff the Output + +```bash +diff -r audit-output/main/ audit-output/feature-system-text-json-v2/ | head -100 +``` + +Or for structured comparison: +```bash +# Compare specific test outputs +diff audit-output/main/events-post-snake-case/elastic.json \ + audit-output/feature-system-text-json-v2/events-post-snake-case/elastic.json +``` + +### 3. Categorize Differences + +Common difference categories: +| Category | Example | Severity | +|----------|---------|----------| +| Casing binding failure | `ReferenceId` in ExtensionData instead of property | CRITICAL | +| Date parsing expansion | `"2026-01-15"` → `"2026-01-15T00:00:00+00:00"` | MEDIUM | +| Numeric precision | `0` vs `0.0` | LOW | +| Empty collection omission | `"tags": []` omitted | LOW/EXPECTED | +| Character encoding | `&` vs `\u0026` | LOW | + +### 4. Write Targeted Tests + +For each difference found, write a **unit test** that reproduces it in isolation: + +```csharp +// In tests/Exceptionless.Tests/Serializer/CasingCompatibilityTests.cs +[Theory] +[InlineData("reference_id")] // snake_case - should always work +[InlineData("ReferenceId")] // PascalCase - must also work +[InlineData("referenceId")] // camelCase - must also work +public void Deserialize_ReferenceId_MatchesAllCasings(string key) +{ + string json = $$"""{"type": "error", "{{key}}": "test-ref-123"}"""; + var ev = _serializer.Deserialize(json); + Assert.Equal("test-ref-123", ev.ReferenceId); +} +``` + +### 5. Implement Fixes + +Common fix patterns: +- **Multi-word property casing:** Add fallback in `IJsonOnDeserialized.OnDeserialized()` to check ExtensionData for alternate casings +- **Date-only string parsing:** Check for time separator ('T') before calling `TryGetDateTimeOffset` in `ObjectToInferredTypesConverter` +- **Naming policy mismatches:** Use `[JsonPropertyName]` attributes or TypeInfo modifiers + +### 6. Re-run Audit + +After fixes, run the audit into a new output directory (or the same branch directory — it overwrites): + +```bash +pwsh .agents/skills/serialization-audit/scripts/audit-api-surface.ps1 -AuditRunId post-fixes +``` + +Compare again to verify differences are resolved. + +## Key Files + +| File | Purpose | +|------|---------| +| `tests/Exceptionless.Tests/Serializer/CasingCompatibilityTests.cs` | Unit tests for specific casing/format issues | +| `src/Exceptionless.Core/Serialization/ObjectToInferredTypesConverter.cs` | Type inference for untyped JSON values | +| `src/Exceptionless.Core/Serialization/JsonSerializerOptionsExtensions.cs` | STJ configuration (naming policy, converters) | +| `src/Exceptionless.Core/Models/Event.cs` | Event model with `OnDeserialized` fallback logic | +| `.agents/skills/serialization-audit/scripts/audit-api-surface.ps1` | Live localhost audit runner | +| `audit-output/` | Generated comparison files (gitignored) | + +## Design Principles + +1. **Backwards compatibility first:** Any payload that worked with Newtonsoft must still work with STJ +2. **Snake_case output, any-case input:** Serialize as snake_case, but accept PascalCase, camelCase, and snake_case on deserialization +3. **Preserve user data types:** Don't expand date-only strings to DateTimeOffset — users may store non-date strings that happen to look like dates +4. **Test the full pipeline:** Unit tests catch the bug, integration tests prove the fix works end-to-end diff --git a/.agents/skills/serialization-audit/scripts/audit-api-surface.ps1 b/.agents/skills/serialization-audit/scripts/audit-api-surface.ps1 new file mode 100644 index 0000000000..8203ad0cef --- /dev/null +++ b/.agents/skills/serialization-audit/scripts/audit-api-surface.ps1 @@ -0,0 +1,764 @@ +#!/usr/bin/env pwsh +<# +.SYNOPSIS + Live API surface audit for Exceptionless serialization testing. + +.DESCRIPTION + Posts events and other entities to a running Exceptionless API using all casing + conventions (snake_case, camelCase, PascalCase, mixed) and captures the raw + HTTP request, API response, and Elasticsearch document for each scenario. + + Output is saved to: audit-output/{AuditRunId}/{BranchName}/{scenario}/ + - request.json : the body sent to the API + - response.json : the API response (GET after processing) + - elastic.json : the raw ES document + +.EXAMPLE + # Run against the running feature branch app (auto-detect branch) + pwsh .agents/skills/serialization-audit/scripts/audit-api-surface.ps1 + + # Run with explicit branch name and run ID + pwsh .agents/skills/serialization-audit/scripts/audit-api-surface.ps1 -BranchName "main" -AuditRunId "live-2026-05-20" + + # Run against a different server + pwsh .agents/skills/serialization-audit/scripts/audit-api-surface.ps1 -BaseUrl "http://localhost:5200" -EsUrl "http://localhost:9201" +#> +param( + [string]$BaseUrl = "http://localhost:7110", + [string]$EsUrl = "http://localhost:9200", + [string]$BranchName = "", # Auto-detected from git if empty + [string]$AuditRunId = "live", + [string]$UserEmail = "admin@exceptionless.test", + [string]$UserPassword = "tester", + [string]$ProjectApiKey = "LhhP1C9gijpSKCslHHCvwdSIz298twx271nTest", + [string]$UserApiKey = "5f8aT5j0M1SdWCMOiJKCrlDNHMI38LjCH4LTTest", + [int]$PollTimeoutSec = 30, + [int]$PollIntervalSec = 2 +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +# ─── Auto-detect branch name ────────────────────────────────────────────────── +if (-not $BranchName) { + try { + $BranchName = (git rev-parse --abbrev-ref HEAD 2>$null).Trim() + } catch { } + if (-not $BranchName) { $BranchName = "unknown" } +} +$BranchName = $BranchName -replace '[/\\]', '-' + +# ─── Output directory ───────────────────────────────────────────────────────── +try { + $RepoRoot = (git -C $PSScriptRoot rev-parse --show-toplevel 2>$null).Trim() +} catch { + $RepoRoot = "" +} + +if (-not $RepoRoot) { + $RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "../../../..")).Path +} + +$OutputDir = Join-Path $RepoRoot "audit-output" $AuditRunId $BranchName +Write-Host "Output dir : $OutputDir" +Write-Host "Branch : $BranchName" +Write-Host "Base URL : $BaseUrl" +Write-Host "ES URL : $EsUrl" + +# ─── Helper: Save a scenario file ───────────────────────────────────────────── +function Save-ScenarioFile { + param([string]$ScenarioName, [string]$FileName, [string]$Content) + $dir = Join-Path $OutputDir $ScenarioName + New-Item -ItemType Directory -Force -Path $dir | Out-Null + $Content | Set-Content -Path (Join-Path $dir $FileName) -Encoding utf8 +} + +# ─── Helper: Pretty-print JSON ──────────────────────────────────────────────── +# Uses python3 json.tool to avoid PowerShell's ConvertFrom-Json date mangling +# (PowerShell auto-parses ISO 8601 strings and re-serializes in local timezone). +function Format-Json { + param([string]$Json) + try { + $result = ($Json | python3 -c "import sys,json; print(json.dumps(json.loads(sys.stdin.read()), indent=2, ensure_ascii=False))" 2>$null) -join "`n" + if ($LASTEXITCODE -eq 0 -and $result) { return $result } + } catch { } + # Fallback: return as-is if python3 unavailable + return $Json +} + +# ─── Helper: Invoke API request with error handling ─────────────────────────── +function Invoke-Api { + param( + [string]$Method, + [string]$Path, + [string]$ApiKey, + [string]$Body = $null, + [hashtable]$Headers = @{} + ) + $uri = "$BaseUrl/api/v2$Path" + $allHeaders = @{ "Authorization" = "Bearer $ApiKey" } + $Headers + + $params = @{ + Method = $Method + Uri = $uri + Headers = $allHeaders + ErrorAction = "SilentlyContinue" + } + if ($Body) { + $params["Body"] = $Body + $params["ContentType"] = "application/json" + } + + try { + $response = Invoke-WebRequest @params -UseBasicParsing + return @{ + StatusCode = $response.StatusCode + Body = $response.Content + Headers = $response.Headers + } + } catch { + # PowerShell 7: HttpResponseException; PS 5: WebException + $statusCode = 0 + $body = "" + try { + if ($_.Exception.Response) { + $statusCode = [int]$_.Exception.Response.StatusCode + } + } catch { } + try { + # PS 7 HttpResponseException has a Body property + if ($_.Exception.PSObject.Properties['Body']) { + $body = $_.Exception.Body + } + } catch { } + if (-not $body) { $body = $_.ErrorDetails.Message } + if (-not $body) { $body = $_.Exception.Message } + return @{ + StatusCode = $statusCode + Body = $body + Headers = @{} + Error = $_.Exception.Message + } + } +} + +# ─── Helper: Poll until event appears by reference ID ───────────────────────── +function Wait-ForEventByRef { + param([string]$ReferenceId) + $elapsed = 0 + while ($elapsed -lt $PollTimeoutSec) { + Start-Sleep -Seconds $PollIntervalSec + $elapsed += $PollIntervalSec + $resp = Invoke-Api -Method GET -Path "/events/by-ref/$ReferenceId" -ApiKey $UserApiKey + if ($resp.StatusCode -eq 200 -and $resp.Body -and $resp.Body.Trim() -ne "[]") { + return $resp.Body + } + } + return $null +} + +# ─── Helper: Query Elasticsearch for a document by ID ──────────────────────── +function Get-EsDocument { + param([string]$IndexPattern, [string]$DocId) + # Step 1: find the concrete index + $searchBody = "{`"query`":{`"ids`":{`"values`":[`"$DocId`"]}},`"size`":1,`"_source`":false}" + $searchUrl = "$EsUrl/$IndexPattern/_search?ignore_unavailable=true&allow_no_indices=true" + try { + $resolveResp = Invoke-WebRequest -Method POST -Uri $searchUrl ` + -Body $searchBody -ContentType "application/json" -UseBasicParsing -ErrorAction SilentlyContinue + if ($resolveResp.StatusCode -eq 200) { + $hits = ($resolveResp.Content | ConvertFrom-Json -Depth 10).hits.hits + if ($hits -and $hits.Count -gt 0) { + $concreteIndex = $hits[0]._index + # Step 2: GET from concrete index + $getUrl = "$EsUrl/$concreteIndex/_doc/$DocId" + $getResp = Invoke-WebRequest -Method GET -Uri $getUrl ` + -UseBasicParsing -ErrorAction SilentlyContinue + if ($getResp.StatusCode -eq 200) { + $source = ($getResp.Content | ConvertFrom-Json -Depth 20)._source + return Format-Json ($source | ConvertTo-Json -Depth 20 -Compress:$false) + } + } + } + } catch { } + return "{`"error`": `"Document $DocId not found in $IndexPattern`"}" +} + +# ─── Helper: Run a full event POST scenario ─────────────────────────────────── +function Invoke-EventScenario { + param([string]$ScenarioName, [string]$RequestJson, [string]$ReferenceId) + Write-Host " → $ScenarioName" -NoNewline + + # 1. Save request + Save-ScenarioFile -ScenarioName $ScenarioName -FileName "request.json" -Content (Format-Json $RequestJson) + + # 2. POST event + $postResp = Invoke-Api -Method POST -Path "/events" -ApiKey $ProjectApiKey -Body $RequestJson + if ($postResp.StatusCode -notin @(200, 201, 202)) { + Write-Host " ✗ POST failed ($($postResp.StatusCode))" + Save-ScenarioFile -ScenarioName $ScenarioName -FileName "response.json" ` + -Content "{`"error`": `"POST failed with status $($postResp.StatusCode)`", `"body`": $($postResp.Body)}" + return + } + + # 3. Poll for processed event + Write-Host " ." -NoNewline + $eventsJson = Wait-ForEventByRef -ReferenceId $ReferenceId + if (-not $eventsJson) { + Write-Host " ✗ timeout waiting for $ReferenceId" + Save-ScenarioFile -ScenarioName $ScenarioName -FileName "response.json" ` + -Content "{`"error`": `"Timeout waiting for event with reference_id=$ReferenceId`"}" + return + } + + # 4. Parse out event id + $events = $eventsJson | ConvertFrom-Json -Depth 10 + $event = if ($events -is [array]) { $events[0] } else { $events } + $eventId = $event.id + + # 5. GET full event by id for response + $getResp = Invoke-Api -Method GET -Path "/events/$eventId" -ApiKey $UserApiKey + if ($getResp.StatusCode -eq 200) { + Save-ScenarioFile -ScenarioName $ScenarioName -FileName "response.json" ` + -Content (Format-Json $getResp.Body) + } else { + Save-ScenarioFile -ScenarioName $ScenarioName -FileName "response.json" ` + -Content (Format-Json $eventsJson) + } + + # 6. Fetch ES document + $esDoc = Get-EsDocument -IndexPattern "*-events-*" -DocId $eventId + Save-ScenarioFile -ScenarioName $ScenarioName -FileName "elastic.json" -Content $esDoc + + Write-Host " ✓ (id=$eventId)" +} + +# HEALTH CHECK +Write-Host "`n[Health Check] $BaseUrl/api/v2/about" +$health = Invoke-Api -Method GET -Path "/about" -ApiKey $UserApiKey +if ($health.StatusCode -ne 200) { + Write-Host "✗ API not reachable (status $($health.StatusCode)). Start the app with 'aspire run' first." -ForegroundColor Red + exit 1 +} +Write-Host "✓ API is up" + +# ─── Login to get a JWT for admin operations (token creation, webhooks) ─────── +Write-Host "[Login] $UserEmail" +$loginBody = "{`"email`":`"$UserEmail`",`"password`":`"$UserPassword`"}" +$loginResp = Invoke-WebRequest -Method POST -Uri "$BaseUrl/api/v2/auth/login" ` + -Body $loginBody -ContentType "application/json" -UseBasicParsing -ErrorAction SilentlyContinue +$JwtToken = "" +if ($loginResp -and $loginResp.StatusCode -eq 200) { + $JwtToken = ($loginResp.Content | ConvertFrom-Json -Depth 5).token + Write-Host "✓ JWT obtained`n" +} else { + Write-Host "⚠ Login failed, some scenarios may fail (token/webhook creation)`n" +} + +# EVENT POST SCENARIOS (1-8) +Write-Host "[Events — POST]" + +Invoke-EventScenario -ScenarioName "events-post-snake-case" -ReferenceId "audit-snake-001" -RequestJson @' +{ + "type": "error", + "message": "Test error with snake_case payload", + "date": "2026-05-20T12:00:00+00:00", + "tags": ["audit", "snake_case"], + "reference_id": "audit-snake-001", + "count": 1, + "value": 42.5, + "geo": "40.7128,-74.0060", + "data": { + "custom_field": "custom_value", + "nested_object": { "inner_key": "inner_value", "inner_number": 123 } + }, + "@user": { + "identity": "user@example.com", + "name": "Test User", + "data": { "plan_name": "premium" } + }, + "@environment": { + "o_s_name": "Windows 11", + "o_s_version": "10.0.22621", + "ip_address": "192.168.1.100", + "machine_name": "AUDIT-MACHINE", + "runtime_version": ".NET 8.0.1", + "processor_count": 8, + "total_physical_memory": 17179869184, + "process_name": "AuditApp" + }, + "@request": { + "client_ip_address": "10.0.0.100", + "http_method": "POST", + "user_agent": "AuditAgent/1.0", + "is_secure": true, + "host": "audit.localhost", + "path": "/api/audit?key=value&other=123", + "port": 443, + "cookies": { "session_id": "abc123" } + }, + "@simple_error": { + "message": "Null reference exception occurred", + "type": "System.NullReferenceException", + "stack_trace": " at Audit.Tests.Run() in AuditTests.cs:line 42" + } +} +'@ + +Invoke-EventScenario -ScenarioName "events-post-camel-case" -ReferenceId "audit-camel-001" -RequestJson @' +{ + "type": "error", + "message": "Test error with camelCase payload", + "date": "2026-05-20T12:00:00+00:00", + "tags": ["audit", "camelCase"], + "referenceId": "audit-camel-001", + "count": 1, + "value": 42.5, + "geo": "40.7128,-74.0060", + "data": { + "customField": "custom_value", + "nestedObject": { "innerKey": "inner_value", "innerNumber": 123 } + }, + "@user": { + "identity": "user@example.com", + "name": "Test User", + "data": { "planName": "premium" } + }, + "@environment": { + "osName": "Windows 11", + "osVersion": "10.0.22621", + "ipAddress": "192.168.1.100", + "machineName": "AUDIT-MACHINE", + "runtimeVersion": ".NET 8.0.1", + "processorCount": 8, + "processName": "AuditApp" + }, + "@request": { + "clientIpAddress": "10.0.0.100", + "httpMethod": "POST", + "userAgent": "AuditAgent/1.0", + "isSecure": true, + "host": "audit.localhost", + "path": "/api/audit", + "port": 443 + }, + "@simple_error": { + "message": "Null reference exception occurred", + "type": "System.NullReferenceException", + "stackTrace": " at Audit.Tests.Run() in AuditTests.cs:line 42" + } +} +'@ + +Invoke-EventScenario -ScenarioName "events-post-pascal-case" -ReferenceId "audit-pascal-001" -RequestJson @' +{ + "Type": "error", + "Message": "Test error with PascalCase payload", + "Date": "2026-05-20T12:00:00+00:00", + "Tags": ["audit", "PascalCase"], + "ReferenceId": "audit-pascal-001", + "Count": 1, + "Value": 42.5, + "Geo": "40.7128,-74.0060", + "Data": { + "CustomField": "custom_value", + "NestedObject": { "InnerKey": "inner_value", "InnerNumber": 123 } + }, + "@user": { + "Identity": "user@example.com", + "Name": "Test User", + "Data": { "PlanName": "premium" } + }, + "@environment": { + "OSName": "Windows 11", + "OSVersion": "10.0.22621", + "IpAddress": "192.168.1.100", + "MachineName": "AUDIT-MACHINE", + "RuntimeVersion": ".NET 8.0.1", + "ProcessorCount": 8, + "ProcessName": "AuditApp" + }, + "@request": { + "ClientIpAddress": "10.0.0.100", + "HttpMethod": "POST", + "UserAgent": "AuditAgent/1.0", + "IsSecure": true, + "Host": "audit.localhost", + "Path": "/api/audit", + "Port": 443 + } +} +'@ + +Invoke-EventScenario -ScenarioName "events-post-mixed-case" -ReferenceId "audit-mixed-001" -RequestJson @' +{ + "type": "error", + "Message": "Test error with mixed casing payload", + "date": "2026-05-20T12:00:00+00:00", + "Tags": ["audit", "mixed"], + "referenceId": "audit-mixed-001", + "Count": 1, + "value": 42.5, + "data": { + "snake_field": "snake_value", + "CamelField": "camel_value", + "PascalField": "pascal_value" + }, + "@user": { + "identity": "user@example.com", + "Name": "Test User" + }, + "@request": { + "ClientIpAddress": "10.0.0.100", + "http_method": "POST", + "UserAgent": "AuditAgent/1.0" + } +} +'@ + +Invoke-EventScenario -ScenarioName "events-post-special-chars" -ReferenceId "audit-special-001" -RequestJson @' +{ + "type": "error", + "message": "Test with special chars: & \"quoted\" & 'single'", + "date": "2026-05-20T12:00:00+00:00", + "reference_id": "audit-special-001", + "data": { + "html_content": "
Hello & World
", + "url_with_params": "https://example.com/path?foo=bar&baz=qux", + "unicode_text": "Hello \u4e16\u754c \u0441\u0432\u0435\u0442 \u0639\u0627\u0644\u0645", + "emoji_text": "Test \ud83d\ude80 emoji", + "newline_text": "Line1\nLine2\tTabbed", + "null_char": "before\u0000after", + "backslash": "C:\\Users\\test\\file.txt" + } +} +'@ + +Invoke-EventScenario -ScenarioName "events-post-numeric-edge-cases" -ReferenceId "audit-numeric-001" -RequestJson @' +{ + "type": "error", + "message": "Test numeric edge cases", + "date": "2026-05-20T12:00:00+00:00", + "reference_id": "audit-numeric-001", + "data": { + "zero_int": 0, + "zero_float": 0.0, + "negative_zero": -0.0, + "large_int": 9007199254740991, + "negative_int": -2147483648, + "double_val": 1.7976931348623157e+308, + "small_double": 5e-324, + "negative_double": -1.23456789e-100, + "int_max": 2147483647, + "one_point_zero": 1.0 + } +} +'@ + +Invoke-EventScenario -ScenarioName "events-post-null-empty" -ReferenceId "audit-null-001" -RequestJson @' +{ + "type": "error", + "message": "Test null and empty values", + "date": "2026-05-20T12:00:00+00:00", + "reference_id": "audit-null-001", + "tags": [], + "data": { + "null_value": null, + "empty_string": "", + "empty_object": {}, + "empty_array": [] + }, + "@user": null +} +'@ + +Invoke-EventScenario -ScenarioName "events-post-date-formats" -ReferenceId "audit-dates-001" -RequestJson @' +{ + "type": "error", + "message": "Test various date formats", + "date": "2026-05-20T12:00:00+00:00", + "reference_id": "audit-dates-001", + "data": { + "iso_full": "2026-01-15T10:30:00.000Z", + "iso_with_offset": "2026-01-15T10:30:00+05:30", + "iso_no_ms": "2026-01-15T10:30:00Z", + "date_only": "2026-01-15", + "time_only": "10:30:00", + "unix_timestamp": 1705312200, + "unix_timestamp_ms": 1705312200000 + } +} +'@ + +# EVENT GET SCENARIOS (9-10) +Write-Host "`n[Events — GET]" + +# events-get-list: submit an event, then GET the events list +$listRefId = "audit-list-001" +Write-Host " → events-get-list" -NoNewline + +$listPostResp = Invoke-Api -Method POST -Path "/events" -ApiKey $ProjectApiKey -Body @' +{ + "type": "log", + "message": "Test list retrieval", + "date": "2026-05-20T12:00:00+00:00", + "reference_id": "audit-list-001", + "source": "AuditSource-List" +} +'@ + +if ($listPostResp.StatusCode -in @(200, 201, 202)) { + $listEventsJson = Wait-ForEventByRef -ReferenceId $listRefId + if ($listEventsJson) { + # GET the events list + $listResp = Invoke-Api -Method GET -Path "/events?limit=10&sort=-date" -ApiKey $UserApiKey + $requestObj = @{ method = "GET"; path = "/api/v2/events"; params = @{ limit = 10; sort = "-date" } } + Save-ScenarioFile -ScenarioName "events-get-list" -FileName "request.json" ` + -Content (Format-Json ($requestObj | ConvertTo-Json -Depth 5)) + Save-ScenarioFile -ScenarioName "events-get-list" -FileName "response.json" ` + -Content (Format-Json $listResp.Body) + + # Also get ES doc for the first event in the list + $events = $listEventsJson | ConvertFrom-Json -Depth 10 + $event = if ($events -is [array]) { $events[0] } else { $events } + $esDoc = Get-EsDocument -IndexPattern "*-events-*" -DocId $event.id + Save-ScenarioFile -ScenarioName "events-get-list" -FileName "elastic.json" -Content $esDoc + Write-Host " ✓" + } else { Write-Host " ✗ timeout" } +} else { Write-Host " ✗ POST failed ($($listPostResp.StatusCode))" } + +# events-get-stack-mode +$stackModeRefId = "audit-stackmode-001" +Write-Host " → events-get-stack-mode" -NoNewline + +$stackModePostResp = Invoke-Api -Method POST -Path "/events" -ApiKey $ProjectApiKey -Body @' +{ + "type": "log", + "message": "Test stack mode retrieval", + "date": "2026-05-20T12:00:00+00:00", + "reference_id": "audit-stackmode-001", + "source": "AuditSource-StackMode" +} +'@ + +if ($stackModePostResp.StatusCode -in @(200, 201, 202)) { + $smEventsJson = Wait-ForEventByRef -ReferenceId $stackModeRefId + if ($smEventsJson) { + $smResp = Invoke-Api -Method GET -Path "/events?mode=stack_new&limit=10" -ApiKey $UserApiKey + $smRequestObj = @{ method = "GET"; path = "/api/v2/events"; params = @{ mode = "stack_new"; limit = 10 } } + Save-ScenarioFile -ScenarioName "events-get-stack-mode" -FileName "request.json" ` + -Content (Format-Json ($smRequestObj | ConvertTo-Json -Depth 5)) + Save-ScenarioFile -ScenarioName "events-get-stack-mode" -FileName "response.json" ` + -Content (Format-Json $smResp.Body) + + $smEvents = $smEventsJson | ConvertFrom-Json -Depth 10 + $smEvent = if ($smEvents -is [array]) { $smEvents[0] } else { $smEvents } + $smStackId = $smEvent.stack_id + if ($smStackId) { + $smStackDoc = Get-EsDocument -IndexPattern "*-stacks-*" -DocId $smStackId + Save-ScenarioFile -ScenarioName "events-get-stack-mode" -FileName "elastic.json" -Content $smStackDoc + } + Write-Host " ✓" + } else { Write-Host " ✗ timeout" } +} else { Write-Host " ✗ POST failed ($($stackModePostResp.StatusCode))" } + +# ORGANIZATION SCENARIOS (11-13) +Write-Host "`n[Organizations]" +$orgId = "537650f3b77efe23a47914f3" + +# organizations-get-by-id +Write-Host " → organizations-get-by-id" -NoNewline +$orgGetReq = @{ method = "GET"; path = "/api/v2/organizations/$orgId" } +Save-ScenarioFile -ScenarioName "organizations-get-by-id" -FileName "request.json" ` + -Content (Format-Json ($orgGetReq | ConvertTo-Json -Depth 5)) +$orgGetResp = Invoke-Api -Method GET -Path "/organizations/$orgId" -ApiKey $UserApiKey +Save-ScenarioFile -ScenarioName "organizations-get-by-id" -FileName "response.json" ` + -Content (Format-Json $orgGetResp.Body) +$orgEsDoc = Get-EsDocument -IndexPattern "*-organizations-*" -DocId $orgId +Save-ScenarioFile -ScenarioName "organizations-get-by-id" -FileName "elastic.json" -Content $orgEsDoc +Write-Host " ✓" + +# organizations-patch-snake-case +Write-Host " → organizations-patch-snake-case" -NoNewline +$orgPatchSnakeBody = @' +{ "name": "Acme Corp (snake patched)" } +'@ +Save-ScenarioFile -ScenarioName "organizations-patch-snake-case" -FileName "request.json" ` + -Content (Format-Json $orgPatchSnakeBody) +$orgPatchSnakeResp = Invoke-Api -Method PATCH -Path "/organizations/$orgId" ` + -ApiKey $UserApiKey -Body $orgPatchSnakeBody +Save-ScenarioFile -ScenarioName "organizations-patch-snake-case" -FileName "response.json" ` + -Content (Format-Json $orgPatchSnakeResp.Body) +$orgSnakeEsDoc = Get-EsDocument -IndexPattern "*-organizations-*" -DocId $orgId +Save-ScenarioFile -ScenarioName "organizations-patch-snake-case" -FileName "elastic.json" -Content $orgSnakeEsDoc +Write-Host " ✓" + +# organizations-patch-camel-case +Write-Host " → organizations-patch-camel-case" -NoNewline +$orgPatchCamelBody = @' +{ "name": "Acme Corp (camel patched)" } +'@ +Save-ScenarioFile -ScenarioName "organizations-patch-camel-case" -FileName "request.json" ` + -Content (Format-Json $orgPatchCamelBody) +$orgPatchCamelResp = Invoke-Api -Method PATCH -Path "/organizations/$orgId" ` + -ApiKey $UserApiKey -Body $orgPatchCamelBody +Save-ScenarioFile -ScenarioName "organizations-patch-camel-case" -FileName "response.json" ` + -Content (Format-Json $orgPatchCamelResp.Body) +$orgCamelEsDoc = Get-EsDocument -IndexPattern "*-organizations-*" -DocId $orgId +Save-ScenarioFile -ScenarioName "organizations-patch-camel-case" -FileName "elastic.json" -Content $orgCamelEsDoc +Write-Host " ✓" + +# PROJECT SCENARIOS (14-15) +Write-Host "`n[Projects]" +$projectId = "537650f3b77efe23a47914f4" + +# projects-get-by-id +Write-Host " → projects-get-by-id" -NoNewline +$projGetReq = @{ method = "GET"; path = "/api/v2/projects/$projectId" } +Save-ScenarioFile -ScenarioName "projects-get-by-id" -FileName "request.json" ` + -Content (Format-Json ($projGetReq | ConvertTo-Json -Depth 5)) +$projGetResp = Invoke-Api -Method GET -Path "/projects/$projectId" -ApiKey $UserApiKey +Save-ScenarioFile -ScenarioName "projects-get-by-id" -FileName "response.json" ` + -Content (Format-Json $projGetResp.Body) +$projEsDoc = Get-EsDocument -IndexPattern "*-projects-*" -DocId $projectId +Save-ScenarioFile -ScenarioName "projects-get-by-id" -FileName "elastic.json" -Content $projEsDoc +Write-Host " ✓" + +# projects-patch-snake-case +Write-Host " → projects-patch-snake-case" -NoNewline +$projPatchBody = @' +{ "name": "My Project (snake patched)" } +'@ +Save-ScenarioFile -ScenarioName "projects-patch-snake-case" -FileName "request.json" ` + -Content (Format-Json $projPatchBody) +$projPatchResp = Invoke-Api -Method PATCH -Path "/projects/$projectId" ` + -ApiKey $UserApiKey -Body $projPatchBody +Save-ScenarioFile -ScenarioName "projects-patch-snake-case" -FileName "response.json" ` + -Content (Format-Json $projPatchResp.Body) +$projPatchEsDoc = Get-EsDocument -IndexPattern "*-projects-*" -DocId $projectId +Save-ScenarioFile -ScenarioName "projects-patch-snake-case" -FileName "elastic.json" -Content $projPatchEsDoc +Write-Host " ✓" + +# STACK SCENARIO (16) +Write-Host "`n[Stacks]" + +# stacks-get-after-event: reuse a previously submitted event's stack +$stackRefId = "audit-stack-001" +Write-Host " → stacks-get-after-event" -NoNewline + +$stackPostResp = Invoke-Api -Method POST -Path "/events" -ApiKey $ProjectApiKey -Body @' +{ + "type": "error", + "message": "Test stack capture", + "date": "2026-05-20T12:00:00+00:00", + "reference_id": "audit-stack-001", + "source": "AuditSource-Stack", + "@simple_error": { + "message": "Stack test exception", + "type": "System.InvalidOperationException", + "stack_trace": " at StackTest.Run() line 1" + } +} +'@ +if ($stackPostResp.StatusCode -in @(200, 201, 202)) { + $stackEventsJson = Wait-ForEventByRef -ReferenceId $stackRefId + if ($stackEventsJson) { + $stackEvents = $stackEventsJson | ConvertFrom-Json -Depth 10 + $stackEvent = if ($stackEvents -is [array]) { $stackEvents[0] } else { $stackEvents } + $stackId = $stackEvent.stack_id + + $stackGetReq = @{ method = "GET"; path = "/api/v2/stacks/$stackId" } + Save-ScenarioFile -ScenarioName "stacks-get-after-event" -FileName "request.json" ` + -Content (Format-Json ($stackGetReq | ConvertTo-Json -Depth 5)) + + $stackGetResp = Invoke-Api -Method GET -Path "/stacks/$stackId" -ApiKey $UserApiKey + Save-ScenarioFile -ScenarioName "stacks-get-after-event" -FileName "response.json" ` + -Content (Format-Json $stackGetResp.Body) + + $stackEsDoc = Get-EsDocument -IndexPattern "*-stacks-*" -DocId $stackId + Save-ScenarioFile -ScenarioName "stacks-get-after-event" -FileName "elastic.json" -Content $stackEsDoc + Write-Host " ✓" + } else { Write-Host " ✗ timeout" } +} else { Write-Host " ✗ POST failed ($($stackPostResp.StatusCode))" } + +# TOKEN SCENARIO (17) +Write-Host "`n[Tokens]" +Write-Host " → tokens-create-and-get" -NoNewline + +$tokenBody = @' +{ "organization_id": "537650f3b77efe23a47914f3", "project_id": "537650f3b77efe23a47914f4", "scopes": ["client"] } +'@ +Save-ScenarioFile -ScenarioName "tokens-create-and-get" -FileName "request.json" ` + -Content (Format-Json $tokenBody) +$tokenAuthKey = if ($JwtToken) { $JwtToken } else { $UserApiKey } +$tokenCreateResp = Invoke-Api -Method POST -Path "/tokens" -ApiKey $tokenAuthKey -Body $tokenBody +if ($tokenCreateResp.StatusCode -in @(200, 201)) { + Save-ScenarioFile -ScenarioName "tokens-create-and-get" -FileName "response.json" ` + -Content (Format-Json $tokenCreateResp.Body) + $tokenId = ($tokenCreateResp.Body | ConvertFrom-Json -Depth 5).id + if ($tokenId) { + $tokenEsDoc = Get-EsDocument -IndexPattern "*-tokens-*" -DocId $tokenId + Save-ScenarioFile -ScenarioName "tokens-create-and-get" -FileName "elastic.json" -Content $tokenEsDoc + } + Write-Host " ✓" +} else { + Save-ScenarioFile -ScenarioName "tokens-create-and-get" -FileName "response.json" ` + -Content "{`"error`": `"Create failed: $($tokenCreateResp.StatusCode)`", `"body`": $($tokenCreateResp.Body)}" + Write-Host " ✗ ($($tokenCreateResp.StatusCode))" +} + +# WEBHOOK SCENARIOS (18-19) +Write-Host "`n[Webhooks]" + +$webhookAuthKey = if ($JwtToken) { $JwtToken } else { $UserApiKey } + +# webhooks-create-snake-case +Write-Host " → webhooks-create-snake-case" -NoNewline +$webhookSnakeBody = @' +{ + "organization_id": "537650f3b77efe23a47914f3", + "project_id": "537650f3b77efe23a47914f4", + "url": "https://audit.localhost/webhook/snake", + "event_types": ["NewError", "NewEvent"] +} +'@ +Save-ScenarioFile -ScenarioName "webhooks-create-snake-case" -FileName "request.json" ` + -Content (Format-Json $webhookSnakeBody) +$webhookSnakeResp = Invoke-Api -Method POST -Path "/webhooks" -ApiKey $webhookAuthKey -Body $webhookSnakeBody +Save-ScenarioFile -ScenarioName "webhooks-create-snake-case" -FileName "response.json" ` + -Content (Format-Json $webhookSnakeResp.Body) +if ($webhookSnakeResp.StatusCode -in @(200, 201)) { + $wId = ($webhookSnakeResp.Body | ConvertFrom-Json -Depth 5).id + if ($wId) { + $whEsDoc = Get-EsDocument -IndexPattern "*-webhooks*" -DocId $wId + Save-ScenarioFile -ScenarioName "webhooks-create-snake-case" -FileName "elastic.json" -Content $whEsDoc + } + Write-Host " ✓" +} else { Write-Host " ✗ ($($webhookSnakeResp.StatusCode))" } + +# webhooks-create-camel-case +Write-Host " → webhooks-create-camel-case" -NoNewline +$webhookCamelBody = @' +{ + "organizationId": "537650f3b77efe23a47914f3", + "projectId": "537650f3b77efe23a47914f4", + "url": "https://audit.localhost/webhook/camel", + "eventTypes": ["NewError", "NewEvent"] +} +'@ +Save-ScenarioFile -ScenarioName "webhooks-create-camel-case" -FileName "request.json" ` + -Content (Format-Json $webhookCamelBody) +$webhookCamelResp = Invoke-Api -Method POST -Path "/webhooks" -ApiKey $webhookAuthKey -Body $webhookCamelBody +Save-ScenarioFile -ScenarioName "webhooks-create-camel-case" -FileName "response.json" ` + -Content (Format-Json $webhookCamelResp.Body) +if ($webhookCamelResp.StatusCode -in @(200, 201)) { + $wId2 = ($webhookCamelResp.Body | ConvertFrom-Json -Depth 5).id + if ($wId2) { + $whEsDoc2 = Get-EsDocument -IndexPattern "*-webhooks*" -DocId $wId2 + Save-ScenarioFile -ScenarioName "webhooks-create-camel-case" -FileName "elastic.json" -Content $whEsDoc2 + } + Write-Host " ✓" +} else { Write-Host " ✗ ($($webhookCamelResp.StatusCode))" } + +# DONE +Write-Host "`n[Done] Output saved to: $OutputDir`n" diff --git a/.agents/skills/thermo-nuclear-code-quality-review/SKILL.md b/.agents/skills/thermo-nuclear-code-quality-review/SKILL.md new file mode 100644 index 0000000000..ac76a2bc88 --- /dev/null +++ b/.agents/skills/thermo-nuclear-code-quality-review/SKILL.md @@ -0,0 +1,192 @@ +--- +name: thermo-nuclear-code-quality-review +description: Run an extremely strict maintainability review for abstraction quality, giant files, and spaghetti-condition growth. Use for a thermo-nuclear code quality review, thermonuclear review, deep code quality audit, or especially harsh maintainability review. +disable-model-invocation: true +--- + +# Thermo-Nuclear Code Quality Review + +Use this skill for an unusually strict review focused on implementation quality, maintainability, abstraction quality, and codebase health. + +Above all, this skill should push the reviewer to be **ambitious** about code structure. Do not merely identify local cleanup opportunities. Actively search for "code judo" moves: restructurings that preserve behavior while making the implementation dramatically simpler, smaller, more direct, and more elegant. + +## Core Prompt + +Start from this baseline: + +> Perform a deep code quality audit of the current branch's changes. +> Rethink how to structure / implement the changes to meaningfully improve code quality without impacting behavior. +> Work to improve abstractions, modularity, reduce Spaghetti code, improve succinctness and legibility. +> Be ambitious, if there is a clear path to improving the implementation that involves restructuring some of the codebase, go for it. +> Be extremely thorough and rigorous. Measure twice, cut once. + +## Non-Negotiable Additional Standards + +Apply the baseline prompt above, plus these explicit review rules: + +0. **Be ambitious about structural simplification.** + - Do not stop at "this could be a bit cleaner." + - Look for opportunities to reframe the change so that whole branches, helpers, modes, conditionals, or layers disappear entirely. + - Prefer the solution that makes the code feel inevitable in hindsight. + - Assume there is often a "code judo" move available: a re-organization that uses the existing architecture more effectively and makes the change dramatically simpler and more elegant. + - If you see a path to delete complexity rather than rearrange it, push hard for that path. + +1. **Do not let a PR push a file from under 1k lines to over 1k lines without a very strong reason.** + - Treat this as a strong code-quality smell by default. + - Prefer extracting helpers, subcomponents, modules, or local abstractions instead of letting a file sprawl past 1000 lines. + - If the diff crosses that threshold, explicitly ask whether the code should be decomposed first. + - Only waive this if there is a compelling structural reason and the resulting file is still clearly organized. + +2. **Do not allow random spaghetti growth in existing code.** + - Be highly suspicious of new ad-hoc conditionals, scattered special cases, or one-off branches inserted into unrelated flows. + - If a change adds "weird if statements in random places", treat that as a design problem, not a stylistic nit. + - Prefer pushing the logic into a dedicated abstraction, helper, state machine, policy object, or separate module instead of tangling an existing path. + - Call out changes that make the surrounding code harder to reason about, even if they technically work. + +3. **Bias toward cleaning the design, not just accepting working code.** + - If behavior can stay the same while the structure becomes meaningfully cleaner, push for the cleaner version. + - Do not rubber-stamp "it works" implementations that leave the codebase messier. + - Strongly prefer simplifications that remove moving pieces altogether over refactors that merely spread the same complexity around. + +4. **Prefer direct, boring, maintainable code over hacky or magical code.** + - Treat brittle, ad-hoc, or "magic" behavior as a code-quality problem. + - Be skeptical of generic mechanisms that hide simple data-shape assumptions. + - Flag thin abstractions, identity wrappers, or pass-through helpers that add indirection without buying clarity. + +5. **Push hard on type and boundary cleanliness when they affect maintainability.** + - Question unnecessary optionality, `unknown`, `any`, or cast-heavy code when a clearer type boundary could exist. + - Prefer explicit typed models or shared contracts over loosely-shaped ad-hoc objects. + - If a branch relies on silent fallback to paper over an unclear invariant, ask whether the boundary should be made explicit instead. + +6. **Keep logic in the canonical layer and reuse existing helpers.** + - Call out feature logic leaking into shared paths or implementation details leaking through APIs. + - Prefer existing canonical utilities/helpers over bespoke one-offs. + - Push code toward the right package, service, or module instead of normalizing architectural drift. + +7. **Treat unnecessary sequential orchestration and non-atomic updates as design smells when the cleaner structure is obvious.** + - If independent work is serialized for no good reason, ask whether the flow should run in parallel instead. + - If related updates can leave state half-applied, push for a more atomic structure. + - Do not over-index on micro-optimizations, but do flag avoidable orchestration complexity that makes the implementation more brittle. + +## Primary Review Questions + +For every meaningful change, ask: + +- Is there a "code judo" move that would make this dramatically simpler? +- Can this change be reframed so fewer concepts, branches, or helper layers are needed? +- Does this improve or worsen the local architecture? +- Did the diff add branching complexity where a better abstraction should exist? +- Did a previously cohesive module become more coupled, more stateful, or harder to scan? +- Is this logic living in the right file and layer? +- Did this change enlarge a file or component past a healthy size boundary? +- Are there repeated conditionals that signal a missing model or missing helper? +- Is the implementation direct and legible, or does it rely on special cases and incidental control flow? +- Is this abstraction actually earning its keep, or is it just a wrapper? +- Did the diff introduce casts, optionality, or ad-hoc object shapes that obscure the real invariant? +- Is this logic living in the canonical layer, or did the diff leak details across a boundary? +- Is this orchestration more sequential or less atomic than it needs to be? + +## What to Flag Aggressively + +Escalate findings when you see: + +- A complicated implementation where a cleaner reframing could delete whole categories of complexity. +- Refactors that move code around but fail to reduce the number of concepts a reader must hold in their head. +- A file crossing 1000 lines due to the PR, especially if the new code could be split out. +- New conditionals bolted onto unrelated code paths. +- One-off booleans, nullable modes, or flags that complicate existing control flow. +- Feature-specific logic leaking into general-purpose modules. +- Generic "magic" handling that hides simple structure and makes the code harder to reason about. +- Thin wrappers or identity abstractions that add indirection without simplifying anything. +- Unnecessary casts, `any`, `unknown`, or optional params that muddy the real contract. +- Copy-pasted logic instead of extracted helpers. +- Narrow edge-case handling implemented in the middle of an already busy function. +- Refactors that technically pass tests but make the code less modular or less readable. +- "Temporary" branching that is likely to become permanent debt. +- Bespoke helpers where the codebase already has a canonical utility for the job. +- Logic added in the wrong layer/package when it should live somewhere more central. +- Sequential async flow where obviously independent work could stay simpler and clearer with parallel execution. +- Partial-update logic that leaves state less atomic than necessary. + +## Preferred Remedies + +When you identify a code-quality problem, prefer suggestions like: + +- Delete a whole layer of indirection rather than polishing it. +- Reframe the state model so conditionals disappear instead of getting centralized. +- Change the ownership boundary so the feature becomes a natural extension of an existing abstraction. +- Turn special-case logic into a simpler default flow with fewer exceptions. +- Extract a helper or pure function. +- Split a large file into smaller focused modules. +- Move feature-specific logic behind a dedicated abstraction. +- Replace condition chains with a typed model or explicit dispatcher. +- Separate orchestration from business logic. +- Collapse duplicate branches into a single clearer flow. +- Delete wrappers that do not meaningfully clarify the API. +- Reuse the existing canonical helper instead of introducing a near-duplicate. +- Make type boundaries more explicit so the control flow gets simpler. +- Move the logic to the package/module/layer that already owns the concept. +- Parallelize independent work when that also simplifies the orchestration. +- Restructure related updates into a more atomic flow when partial state would be harder to reason about. + +Do not be satisfied with "maybe rename this" feedback when the real issue is structural. +Do not be satisfied with a merely cleaner version of the same messy idea if there is a plausible path to a much simpler idea. + +## Review Tone + +Be direct, serious, and demanding about quality. +Do not be rude, but do not soften major maintainability issues into mild suggestions. +If the code is making the codebase messier, say so clearly. +If the implementation missed an opportunity for a dramatic simplification, say that clearly too. + +Good phrases: + +- `this pushes the file past 1k lines. can we decompose this first?` +- `this adds another special-case branch into an already busy flow. can we move this behind its own abstraction?` +- `this works, but it makes the surrounding code more spaghetti. let's keep the behavior and restructure the implementation.` +- `this feels like feature logic leaking into a shared path. can we isolate it?` +- `this abstraction seems unnecessary. can we just keep the direct flow?` +- `why does this need a cast / optional here? can we make the boundary more explicit instead?` +- `this looks like a bespoke helper for something we already have elsewhere. can we reuse the canonical one?` +- `i think there's a code-judo move here that makes this much simpler. can we reframe this so these branches disappear?` +- `this refactor moves complexity around, but doesn't really delete it. is there a way to make the model itself simpler?` + +## Output Expectations + +Prioritize findings in this order: + +1. Structural code-quality regressions +2. Missed opportunities for dramatic simplification / code-judo restructuring +3. Spaghetti / branching complexity increases +4. Boundary / abstraction / type-contract problems that make the code harder to reason about +5. File-size and decomposition concerns +6. Modularity and abstraction issues +7. Legibility and maintainability concerns + +Do not flood the review with low-value nits if there are larger structural issues. +Prefer a smaller number of high-conviction comments over a long list of cosmetic notes. + +## Approval Bar + +Do not approve merely because behavior seems correct. +The bar for approval is: + +- no clear structural regression +- no obvious missed opportunity to make the implementation dramatically simpler when such a path is visible +- no unjustified file-size explosion +- no obvious spaghetti-growth from special-case branching +- no obviously hacky or magical abstraction that makes the code harder to reason about +- no unnecessary wrapper/cast/optionality churn obscuring the real design +- no clear architecture-boundary leak or avoidable canonical-helper duplication +- no missed opportunity for an obvious decomposition that would materially improve maintainability + +Treat these as presumptive blockers unless the author can justify them clearly: + +- the PR preserves a lot of incidental complexity when there is a plausible code-judo move that would delete it +- the PR pushes a file from below 1000 lines to above 1000 lines +- the PR adds ad-hoc branching that makes an existing flow more tangled +- the PR solves a local problem by scattering feature checks across shared code +- the PR adds an unnecessary abstraction, wrapper, or cast-heavy contract that makes the design more indirect +- the PR duplicates an existing helper or puts logic in the wrong layer when there is a clear canonical home + +If those conditions are not met, leave explicit, actionable feedback and push for a cleaner decomposition. diff --git a/.claude/agents/thermo-nuclear-code-quality-review.md b/.claude/agents/thermo-nuclear-code-quality-review.md new file mode 100644 index 0000000000..dc83d95930 --- /dev/null +++ b/.claude/agents/thermo-nuclear-code-quality-review.md @@ -0,0 +1,23 @@ +--- +name: thermo-nuclear-code-quality-review +description: Thermo-nuclear code quality audit (maintainability, structure, 1k-line rule, spaghetti, code-judo). Invoked via Task after a parent gathers diff and file contents. Loads the rubric from the `thermo-nuclear-code-quality-review` skill in the cursor-team-kit plugin. +--- + +# Thermo-Nuclear Code Quality Review + +You are a **Task subagent**. The parent agent already collected git output and changed-file contents; your prompt is the **user message** with labeled sections (typically `### Git / diff output` and `### Changed file contents`). + +## Rubric + +1. Load the `thermo-nuclear-code-quality-review` skill (shipped in the cursor-team-kit plugin) and treat its `SKILL.md` as the **complete** rubric — tone, approval bar, output ordering, code-judo / 1k-line / spaghetti rules. +2. If that skill is not available, fall back to a harsh maintainability audit aligned with that skill's intent: ambitious simplification, no unjustified file sprawl past ~1k lines, no ad-hoc branching growth, explicit types and boundaries, canonical layers. + +## Work + +- Apply the rubric **only** to what the diff and contents show. Trace cross-file impact when the change touches module boundaries. +- Output in the **priority order** the rubric specifies. Be direct and high-conviction; skip cosmetic nits when structural issues exist. +- Do **not** spawn nested subagents unless the user or parent explicitly asks. + +## Parent orchestration + +Typical flow: in **one** message, run two `Task` calls in parallel — `subagent_type: "shell"` and `subagent_type: "explore"` — to collect `git diff ...HEAD` output and full contents of changed files (default base `main`). Then invoke this agent with `subagent_type: "thermo-nuclear-code-quality-review"` and a user prompt containing `### Git / diff output` and `### Changed file contents`. diff --git a/.gitignore b/.gitignore index 5cd0861747..1ab442acbd 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ artifacts [Tt]est[Rr]esult*/ [Bb]uild[Ll]og.* coverage/ +audit-output/ src/Exceptionless.Web/ClientApp/test-results/ # IDE / editor @@ -80,7 +81,6 @@ tmpclaude* dogfood-output/ debug-storybook.log - .devcontainer/devcontainer-lock.json *.lscache diff --git a/AGENTS.md b/AGENTS.md index 7e2c596d42..7ee86243a3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -59,6 +59,7 @@ Available in `.claude/agents/`. Use `@agent-name` to invoke: - Prefer additive documentation updates — don't replace strategic docs wholesale, extend them - **Backwards compatibility:** Never break existing public APIs, WebSocket message formats, config keys, or exported library interfaces without explicit user approval. Call out any breaking change as a BLOCKER in reviews. - **API test files:** Update `tests/http/*.http` files whenever endpoints change (new, modified, or removed). +- **Abbreviations:** Never abbreviate `Organization` as `org` in code (variable names, parameters, method names, or comments). Always spell out `organization`. - **PR descriptions:** When creating a PR, fill out any existing PR template. Provide concise context: what changed, why, new APIs/features/behaviors, and any breaking changes. No essays — just enough for reviewers to understand the value and impact. - **App URL for QA:** `http://localhost:7110` — probe `/api/v2/about` for health check. - **Never test against production:** Always dogfood, QA test, and run API smoke tests against `localhost` only. Never use production URLs (e.g., `be.exceptionless.io`) in scripts, tests, or browser automation. Start the app locally via `aspire run` or the AppHost before testing. @@ -102,3 +103,28 @@ When upgrading dependencies (applies to implementation and review): ## Frontend Notes - Saved-view optimistic writes must update both `queryKeys.view(organizationId, view)` and `queryKeys.organization(organizationId)` caches immediately. `invalidateSavedViewQueries` delays `SavedViewChanged` `Added` and `Saved` WebSocket invalidations for Elasticsearch refresh safety, and the picker still uses local 1.5s invalidation timers for rename/default/delete flows. + +## Serialization Architecture + +The project uses **System.Text.Json (STJ)** exclusively. The Elasticsearch repository stack uses `Elastic.Clients.Elasticsearch`; application-level serialization should not depend on Newtonsoft.Json/NEST types: + +| Component | Serializer / API | Notes | +| -------------- | --------------------------------- | ------------------------------------------------------------- | +| Elasticsearch | `DefaultSourceSerializer` | Configured in `ExceptionlessElasticConfiguration` with STJ | +| Event Upgrader | `System.Text.Json.Nodes` | JsonObject/JsonArray for mutable DOM | +| Data Storage | `SystemTextJsonSerializer` | Via Foundatio's STJ support | +| API | STJ (built-in) | ASP.NET Core default with Exceptionless serializer options | + +**Key files:** + +- `ExceptionlessElasticConfiguration.cs` - Elasticsearch client and source serializer setup +- `JsonSerializerOptionsExtensions.cs` - Shared STJ naming, encoder, converter, and resolver defaults +- `JsonNodeExtensions.cs` - STJ equivalents of JObject helpers +- `ObjectToInferredTypesConverter.cs` - Infers native .NET types for `object`-typed JSON values +- `JsonElementConverter.cs` - Converts captured `JsonElement` extension data into native .NET values +- `V*_EventUpgrade.cs` - Event version upgraders using JsonObject + +**Security:** + +- Safe JSON encoding used everywhere (escapes `<`, `>`, `&`, `'` for XSS protection) +- No `UnsafeRelaxedJsonEscaping` in the codebase diff --git a/docs/serialization-architecture.md b/docs/serialization-architecture.md new file mode 100644 index 0000000000..c835859a9d --- /dev/null +++ b/docs/serialization-architecture.md @@ -0,0 +1,459 @@ +# Serialization Architecture + +This document describes the complete serialization architecture after the Newtonsoft.Json → System.Text.Json (STJ) migration. It covers every serialization path, data transformation, naming convention, and compatibility consideration. + +## Table of Contents + +1. [Serializer Configuration](#serializer-configuration) +2. [Serialization Paths](#serialization-paths) +3. [Data Flow: Event Lifecycle](#data-flow-event-lifecycle) +4. [GetValue\ Dictionary Extraction](#getvaluet-dictionary-extraction) +5. [ObjectToInferredTypesConverter](#objecttoinferredtypesconverter) +6. [Event Upgrade Pipeline](#event-upgrade-pipeline) +7. [Model Annotations & Naming](#model-annotations--naming) +8. [Elasticsearch Divergences](#elasticsearch-divergences) +9. [Transitive Dependencies](#transitive-dependencies) +10. [Production Safety Guarantees](#production-safety-guarantees) + +--- + +## Serializer Configuration + +### Primary Configuration (`ConfigureExceptionlessDefaults`) + +**File:** `src/Exceptionless.Core/Serialization/JsonSerializerOptionsExtensions.cs` + +All serialization in the app starts from a single extension method that configures `JsonSerializerOptions`: + +```csharp +options.DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull; +options.PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower; +options.PropertyNameCaseInsensitive = true; +options.Encoder = JavaScriptEncoder.Create(UnicodeRanges.All); // XSS-safe +options.Converters.Add(new ObjectToInferredTypesConverter()); +options.IncludeFields = true; +options.RespectNullableAnnotations = true; +options.TypeInfoResolver = new DefaultJsonTypeInfoResolver +{ + Modifiers = { EmptyCollectionModifier.SkipEmptyCollections } +}; +``` + +**Key behaviors:** +- **Naming:** All properties serialize as `snake_case_lower` (e.g., `LastOccurrence` → `"last_occurrence"`) +- **Nulls:** Null properties are omitted from output +- **Empty collections:** Empty `[]` and `{}` are omitted (matches Newtonsoft behavior) +- **Case-insensitive deserialization:** Reads both `"stack_trace"` and `"StackTrace"` for the same property +- **XSS-safe encoding:** `<`, `>`, `&`, `'` are escaped in all JSON output + +### DI Registration + +**File:** `src/Exceptionless.Core/Bootstrapper.cs` + +``` +JsonSerializerOptions (singleton) → ConfigureExceptionlessDefaults() + ↓ +ITextSerializer (singleton) → SystemTextJsonSerializer(options) + ↓ +ISerializer (alias) → same instance +``` + +Every Foundatio infrastructure component (queues, cache, message bus) resolves `ISerializer` from DI and gets the STJ-backed serializer. + +--- + +## Serialization Paths + +### Path 1: API Responses (ASP.NET Core) + +**Config:** `Startup.cs` → `.AddJsonOptions(o => o.JsonSerializerOptions.ConfigureExceptionlessDefaults())` + +- Separate `JsonSerializerOptions` instance from DI, but identically configured +- Additional converter: `DeltaJsonConverterFactory` for PATCH operations +- Also configured for Minimal APIs: `.ConfigureHttpJsonOptions(...)` + +### Path 2: Elasticsearch Documents + +**Config:** `ExceptionlessElasticConfiguration.CreateElasticClient()` → `DefaultSourceSerializer` + +Uses `ConfigureExceptionlessDefaults()` + `ConfigureFoundatioRepositoryDefaults()` with these **overrides**: + +| Setting | API/App | Elasticsearch | Reason | +|---------|---------|---------------|--------| +| `RespectNullableAnnotations` | `true` | `false` | Legacy ES data has unexpected nulls | +| `ObjectToInferredTypesConverter` | `preferInt64: false` | `preferInt64: true` | ES maps numbers as long | +| `JsonStringEnumConverter` | Registered (from Foundatio) | **Removed** | Most enums stored as integers in ES | + +**Critical implication:** A number `42` in Event.Data: +- In API response: serialized as `42` (int) +- In Elasticsearch: stored as `42L` (long via preferInt64) +- Both round-trip correctly because deserialization handles both types + +### Path 3: Queue Messages (Redis/Azure/SQS) + +**Config:** Inherits DI `ISerializer` → `SystemTextJsonSerializer` + +Queue payloads (e.g., `EventPost`, `WebHookNotification`, `WorkItemData`) are serialized with the standard options. Messages produced before the migration used Newtonsoft via `Foundatio.JsonNet`. **During rolling deploy, old messages in queues will be deserialized by STJ** — this works because: +- `PropertyNameCaseInsensitive = true` reads both PascalCase and snake_case +- Queue message types are simple DTOs without complex nested data + +### Path 4: Message Bus (Redis Pub/Sub) + +**Config:** Inherits DI `ISerializer` → `SystemTextJsonSerializer` + +Messages: `EntityChanged`, `PlanChanged`, `UserMembershipChanged`, `ReleaseNotification`, `SystemNotification`. All are simple DTOs that serialize cleanly. + +### Path 5: Cache (Redis/InMemory) + +**Config:** Inherits DI `ISerializer` → `SystemTextJsonSerializer` + +Cached values: Organizations, Projects, Stacks, Tokens, Users. Cache is ephemeral — keys expire. No migration needed; old cached values are simply evicted. + +### Path 6: WebSocket Messages + +**Config:** `WebSocketConnectionManager` resolves `ITextSerializer` from DI. + +Messages sent to browser clients use `serializer.SerializeToString(message)` with the standard snake_case options. The JavaScript/TypeScript frontend expects snake_case. + +### Path 7: Webhook Payloads + +**Config:** `WebHooksJob` uses DI `JsonSerializerOptions` → `PostAsJsonAsync(url, data, options)` + +Webhook payloads are serialized as snake_case JSON. This matches the previous behavior (Newtonsoft used `LowerCaseUnderscorePropertyNamesContractResolver`). + +### Path 8: Email Templates + +**Config:** `Mailer` uses DI `ITextSerializer` for model data extraction. + +Event data is extracted via `GetValue()` for building email template models. No direct JSON serialization for the template rendering. + +--- + +## Data Flow: Event Lifecycle + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ 1. INGESTION (EventPostsJob) │ +│ HTTP POST body → JsonSerializer.Deserialize │ +│ Options: ConfigureExceptionlessDefaults() │ +│ • Unknown JSON fields → ExtensionData (JsonElement dict) │ +│ • IJsonOnDeserialized.OnDeserialized() merges into Data │ +│ • ObjectToInferredTypesConverter: objects → Dict │ +└───────────────────────────┬─────────────────────────────────────┘ + │ +┌───────────────────────────▼─────────────────────────────────────┐ +│ 2. PIPELINE PROCESSING │ +│ EventProcessor → Plugin chain (Error, SimpleError, Request, │ +│ Environment, Geo, Session, Angular, Privacy) │ +│ • Reads typed data via GetValue(key, serializer) │ +│ • Mutates (e.g., SetTargetInfo, strip PII) │ +│ • Writes back via Data[key] = mutatedObject │ +└───────────────────────────┬─────────────────────────────────────┘ + │ +┌───────────────────────────▼─────────────────────────────────────┐ +│ 3. ELASTICSEARCH STORAGE │ +│ Repository.SaveAsync(event) │ +│ Options: ConfigureExceptionlessDefaults() + ES overrides │ +│ • preferInt64: true (all ints stored as long) │ +│ • No JsonStringEnumConverter (enums as integers) │ +│ • RespectNullableAnnotations: false │ +│ • EmptyCollectionModifier omits empty arrays/dicts │ +└───────────────────────────┬─────────────────────────────────────┘ + │ +┌───────────────────────────▼─────────────────────────────────────┐ +│ 4. API RESPONSE │ +│ Repository.GetByIdAsync() → ES deserializes → C# model │ +│ Controller returns model → ASP.NET serializes to response │ +│ Options: ConfigureExceptionlessDefaults() │ +│ • Client sees snake_case JSON │ +│ • Data dict keys preserved as-is from ES │ +│ • Numbers: int/long depending on value │ +└─────────────────────────────────────────────────────────────────┘ +``` + +### Extension Data Merge (Ingestion) + +When an event is posted with known data keys at the root level (legacy clients): + +```json +{"type": "error", "@error": {...}, "@request": {...}, "custom_field": "value"} +``` + +STJ deserializes known properties (`type`), then captures unknown keys (`@error`, `@request`, `custom_field`) in `ExtensionData` as `Dictionary`. After deserialization, `OnDeserialized()` merges them into `Data` using `ObjectToInferredTypesConverter.ConvertJsonElement()`: + +- Objects → `Dictionary` (case-insensitive keys) +- Arrays → `List` +- Strings → `string` (with DateTimeOffset detection for ISO 8601) +- Numbers → `int`/`long`/`decimal` +- Booleans → `bool` + +--- + +## GetValue\ Dictionary Extraction + +**File:** `src/Exceptionless.Core/Extensions/DataDictionaryExtensions.cs` + +The `GetValue()` method extracts typed values from `DataDictionary` (which stores `object?` values). After ES round-trip, values in `Data` are `Dictionary` (from ObjectToInferredTypesConverter). + +### Extraction Strategy + +``` +Data["@error"] → Dictionary (in-memory) + ↓ Serialize to JSON with appropriate options + ↓ Deserialize as T via ITextSerializer + = Error object with all properties populated +``` + +### Serialization Options + +The method uses a single serialization options set for the **dictionary→JSON serialize step**: + +```csharp +s_dictSerializeOptions = { + PropertyNamingPolicy = SnakeCaseLower, + DefaultIgnoreCondition = WhenWritingNull +}; +``` + +**Key design decisions:** +- `PropertyNamingPolicy = SnakeCaseLower` converts C# property names to snake_case when serializing typed objects nested within dictionaries +- **No `DictionaryKeyPolicy`** — user-provided dictionary keys (e.g., `Error.Data`, `QueryString`) are preserved exactly as-is +- `PropertyNameCaseInsensitive = true` on the main deserializer handles matching snake_case keys back to PascalCase C# properties + +### Why No DictionaryKeyPolicy + +`DictionaryKeyPolicy` applies recursively to ALL dictionary keys at ALL nesting levels, which would corrupt user-provided data: + +``` +// User submits Error.Data with key "SomeProp" +// With DictionaryKeyPolicy: "SomeProp" → "some_prop" — DATA CORRUPTION +// Without DictionaryKeyPolicy: "SomeProp" preserved as-is ✓ +``` + +In production, ES stores typed property names as snake_case (from `PropertyNamingPolicy`). When `GetValue` deserializes from this data, `PropertyNameCaseInsensitive` handles the snake_case→PascalCase property matching. Dictionary keys don't need normalization because they're user data, not C# property names. + +--- + +## ObjectToInferredTypesConverter + +**File:** `src/Exceptionless.Core/Serialization/ObjectToInferredTypesConverter.cs` + +Custom `JsonConverter` that replaces STJ's default behavior of deserializing `object`-typed properties as `JsonElement`. + +### Type Inference Rules + +| JSON Token | API Mode (preferInt64: false) | ES Mode (preferInt64: true) | +|------------|-------------------------------|------------------------------| +| `true`/`false` | `bool` | `bool` | +| Integer (fits int32) | `int` | `long` | +| Integer (fits int64) | `long` | `long` | +| Float/decimal | `decimal` | `double` | +| ISO 8601 string | `DateTimeOffset` | `DateTimeOffset` | +| DateTime string | `DateTime` | `DateTime` | +| Other string | `string` | `string` | +| `null` | `null` | `null` | +| Object `{}` | `Dictionary` (OrdinalIgnoreCase) | Same | +| Array `[]` | `List` | Same | + +### Number Representation Integrity + +The converter checks raw JSON bytes for decimal points (`'.'`) and exponents (`'e'`/`'E'`). A value like `0.0` stays as `double`/`decimal`, never coerced to `0L`. + +### Static ConvertJsonElement Helper + +Used by `Event.OnDeserialized()` to convert `JsonElement` values from `[JsonExtensionData]` into inferred .NET types. Matches the same rules as the converter but operates on a pre-read `JsonElement` rather than a `Utf8JsonReader`. + +--- + +## Event Upgrade Pipeline + +Events from older clients are upgraded to the current format via JsonNode manipulation. + +### Upgrade Chain + +``` +GetVersion → determines event version from JSON + ↓ +V1R500_EventUpgrade (version ≤ 1.0.0-r500) + • Renames Error.ExtraData → Error.Data +V1R844_EventUpgrade (version ≤ 1.0.0-r844) + • Moves error info from root into structured Error object + • Renames InnerException → Inner + • Processes exception info +V1R850_EventUpgrade (version ≤ 1.0.0-r850) + • Renames RequestInfo properties +V2_EventUpgrade (version ≤ 2.0) + • Complete restructure: flat format → nested Event format + • Moves ExceptionlessClientInfo → @submission_client + • Moves @User → @user_description + @user (as typed objects) + • Creates @error structure from root Code/Type/Inner/StackTrace/TargetMethod + • Handles 404 events (type: "404" vs type: "error") + • Renames ExtendedData → Data + • Processes __ExceptionInfo extra properties +``` + +### STJ Implementation Notes + +All upgraders operate on `JsonNode` (`JsonObject`/`JsonArray`/`JsonValue`) instead of Newtonsoft's `JObject`/`JToken`. Key differences: +- `JsonNode` children must be **detached** before adding to another parent (no implicit cloning) +- `V2_EventUpgrade` uses `JsonSerializer.SerializeToNode(new UserDescription(...))` for typed → node conversion (no options needed for simple DTOs — snake_case isn't required here because the root Event serializer handles the final format) +- Parse uses default `JsonNodeOptions` (max depth 64 from STJ default) + +--- + +## Model Annotations & Naming + +### SnakeCaseLower Naming Policy + +STJ's `JsonNamingPolicy.SnakeCaseLower` converts property names: +- `LastOccurrence` → `last_occurrence` +- `StackTrace` → `stack_trace` +- `IPAddress` → `ip_address` + +### JsonPropertyName Overrides (Legacy Compatibility) + +Some properties need names that differ from what `SnakeCaseLower` would produce: + +| Model | Property | Override | Why | +|-------|----------|----------|-----| +| `EnvironmentInfo` | `OSName` | `"o_s_name"` | Legacy: Newtonsoft produced `o_s_name` (letter-by-letter), not `os_name` | +| `EnvironmentInfo` | `OSVersion` | `"o_s_version"` | Same — preserves ES mapping compatibility | + +### SlackToken Model (External API) + +Slack API requires specific JSON property names. All properties have explicit `[JsonPropertyName]` to match Slack's API contract regardless of our naming policy. + +### StackStatus Enum + +```csharp +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum StackStatus +{ + [JsonStringEnumMemberName("open")] Open = 0, + [JsonStringEnumMemberName("fixed")] Fixed = 10, + [JsonStringEnumMemberName("regressed")] Regressed = 20, + [JsonStringEnumMemberName("snoozed")] Snoozed = 30, + [JsonStringEnumMemberName("ignored")] Ignored = 40, + [JsonStringEnumMemberName("discarded")] Discarded = 50 +} +``` + +The type-level `[JsonConverter(typeof(JsonStringEnumConverter))]` ensures this enum ALWAYS serializes as a string (even in ES where the global JsonStringEnumConverter is removed). This is mapped as a `Keyword` field in the StackIndex. + +### EmptyCollectionModifier + +Omits empty collections and dictionaries from serialized output: +- `Tags: []` → omitted +- `References: []` → omitted +- `Data: {}` → omitted + +This matches the Newtonsoft behavior and keeps ES documents compact. + +--- + +## Elasticsearch Divergences + +### Why ES Serialization Differs + +Elasticsearch stores documents long-term. The ES serializer must: +1. **Store integers as long** — ES maps `number` fields to `long`. If we stored `int`, reading back would fail for values that ES promotes to long internally. +2. **Allow nulls in legacy data** — Old documents may have null in non-nullable properties. `RespectNullableAnnotations = false` prevents deserialization failures. +3. **Store enums as integers** — Most enums (EventType, etc.) are stored as integer values in ES indices. Exception: `StackStatus` uses its own type-level converter. + +### Index Mappings + +Dynamic templates handle user-provided indexed data (`idx.*`): +``` +*-b → boolean +*-d → date +*-n → double +*-r → keyword (reference, max 256 chars) +*-s → keyword (string, max 1024 chars) +``` + +### Rolling Deploy Safety + +During deployment where old instances use Newtonsoft and new instances use STJ: +- **ES documents:** Both serializers produce compatible snake_case JSON. STJ reads old documents fine (`PropertyNameCaseInsensitive = true`). +- **Queue messages:** STJ can read Newtonsoft-produced messages (case-insensitive + simple DTOs). +- **Cache:** Redis cache is ephemeral with TTL. Old entries expire naturally. +- **Message bus:** Pub/sub is real-time. Brief incompatibility window is self-healing. + +--- + +## Transitive Dependencies + +### Newtonsoft.Json (Transitive Only — NOT USED) + +``` +Foundatio.Repositories.Elasticsearch v8.0.0-beta1 + → Foundatio.Repositories v8.0.0-beta1 + → Foundatio.JsonNet v13.0.1 + → Newtonsoft.Json v13.0.4 + +Stripe.net v51.1.0 + → Newtonsoft.Json v13.0.4 +``` + +**Impact:** +- `Foundatio.JsonNet` is a **transitive dependency only**. Our DI explicitly registers `SystemTextJsonSerializer` as `ITextSerializer`/`ISerializer`, overriding any default Foundatio would use. +- `Stripe.net` uses Newtonsoft internally for Stripe API communication. This is isolated to Stripe SDK internals and doesn't affect our serialization. +- No source file in `src/` references `Foundatio.JsonNet`, `JsonNetSerializer`, or uses Newtonsoft types. + +### Why They're Still Present + +These packages will remain until: +- Foundatio.Repositories removes its Foundatio.JsonNet dependency (tracked upstream) +- Stripe.net drops its Newtonsoft.Json dependency. As of v51.1.0 (latest stable), Stripe.net still depends on Newtonsoft.Json directly across all target frameworks (net6.0, net8.0, net9.0). While Stripe added STJ support, they have not yet removed the Newtonsoft dependency. + +Neither impacts our runtime serialization. + +--- + +## Production Safety Guarantees + +### Verified Round-Trip Paths + +All 1549 tests pass, covering: + +1. **Event ingestion → ES → API response** (EventPipelineTests, EventControllerTests) +2. **Error/SimpleError/EnvironmentInfo/RequestInfo extraction** (GetValue with typed deserialization) +3. **WebHook payload generation** (WebHookDataTests with real event fixtures) +4. **Event upgrade V1→V2** (EventUpgraderTests with historical JSON fixtures) +5. **Stack/Organization/Project/Token CRUD** (Repository tests with ES) +6. **Aggregation queries** (AggregationTests) +7. **Session management** (SessionPlugin, CloseInactiveSessionsJob) +8. **Serializer round-trip** (SerializerTests — every model type) + +### Data Mutation Audit + +Every code path that calls `GetValue()`, mutates the result, and needs to persist the change has been verified to write back: + +| Plugin | Mutation | Write-back | +|--------|----------|------------| +| ErrorPlugin | `SetTargetInfo()` | `Data[Error] = error` ✓ | +| SimpleErrorPlugin | `SetTargetInfo()` | `Data[SimpleError] = error` ✓ | +| AngularPlugin | `SetTargetInfo()` | `Data[Error] = error` ✓ | +| RequestInfoPlugin | Strip PII, apply exclusions | `AddRequestInfo(request)` ✓ | +| EnvironmentInfoPlugin | Strip IP/machine name | `SetEnvironmentInfo(env)` ✓ | +| RemovePrivateInformationPlugin | Clear email | `SetUserDescription(desc)` ✓ | + +### No Data Loss Scenarios + +| Scenario | Safety | +|----------|--------| +| Existing ES documents | Read fine — `PropertyNameCaseInsensitive` handles any key format | +| In-flight queue messages | STJ reads Newtonsoft output (case-insensitive DTOs) | +| Cached values | Ephemeral with TTL, auto-expire | +| WebSocket messages | Consumers expect snake_case (unchanged) | +| Webhook consumers | snake_case output (unchanged from Newtonsoft era) | +| Old client submissions | Event upgrader handles V1/V2 format | +| Custom data keys in Event.Data | Preserved as-is (no DictionaryKeyPolicy in main serializer) | + +### Known Limitations + +1. **StackRepository `"open"` magic string** — Foundatio's `FieldValueHelper.ToFieldValue` doesn't yet respect `[JsonStringEnumMemberName]`. Uses `"open"` string literal instead of `StackStatus.Open`. Tracked as a TODO; functionally correct. + +2. **EventRepository `ElasticFilter`** — One remaining raw ES query for `MustNot ExistsQuery` on dynamic index field (`idx.{SessionEnd}-d`). No typed Foundatio alternative exists for dynamic template fields. + +3. **GetValue\ preserves dictionary keys** — `GetValue()` uses `PropertyNamingPolicy = SnakeCaseLower` (for typed property names) but no `DictionaryKeyPolicy`, so user-provided dictionary keys in `Error.Data`, `QueryString`, etc. are preserved exactly as submitted. diff --git a/src/Exceptionless.AppHost/Exceptionless.AppHost.csproj b/src/Exceptionless.AppHost/Exceptionless.AppHost.csproj index 3589e46ecf..5c307f3adf 100644 --- a/src/Exceptionless.AppHost/Exceptionless.AppHost.csproj +++ b/src/Exceptionless.AppHost/Exceptionless.AppHost.csproj @@ -12,7 +12,7 @@ - + diff --git a/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs b/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs index f98122d584..03b989ba64 100644 --- a/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs +++ b/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs @@ -1,4 +1,4 @@ -using HealthChecks.Elasticsearch; +using Elastic.Clients.Elasticsearch; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Diagnostics.HealthChecks; @@ -27,7 +27,6 @@ public static IResourceBuilder AddElasticsearch(this IDis var elasticsearch = new ElasticsearchResource(name); string? connectionString = null; - ElasticsearchOptions? options = null; builder.Eventing.Subscribe(elasticsearch, async (@event, ct) => { @@ -36,16 +35,13 @@ public static IResourceBuilder AddElasticsearch(this IDis { throw new DistributedApplicationException($"ConnectionStringAvailableEvent was published for the '{elasticsearch.Name}' resource but the connection string was null."); } - - options = new ElasticsearchOptions(); - options.UseServer(connectionString); }); string healthCheckKey = $"{name}_check"; builder.Services.AddHealthChecks() .Add(new HealthCheckRegistration( healthCheckKey, - sp => new ElasticsearchHealthCheck(options!), + sp => new ElasticsearchConnectionHealthCheck(() => connectionString), failureStatus: null, tags: null, timeout: null)); @@ -127,3 +123,20 @@ internal static class ElasticsearchContainerImageTags public const string KibanaImage = "kibana/kibana"; public const string Tag = "8.19.15"; } + +internal sealed class ElasticsearchConnectionHealthCheck(Func connectionStringFactory) : IHealthCheck +{ + public async Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) + { + var connectionString = connectionStringFactory(); + if (string.IsNullOrEmpty(connectionString)) + return new HealthCheckResult(context.Registration.FailureStatus, "Connection string not available."); + + using var settings = new ElasticsearchClientSettings(new Uri(connectionString)); + var client = new ElasticsearchClient(settings); + var response = await client.PingAsync(cancellationToken); + return response.IsValidResponse + ? HealthCheckResult.Healthy() + : new HealthCheckResult(context.Registration.FailureStatus, $"Elasticsearch ping failed: {response.DebugInformation}"); + } +} diff --git a/src/Exceptionless.Core/Bootstrapper.cs b/src/Exceptionless.Core/Bootstrapper.cs index 7950552401..206e346652 100644 --- a/src/Exceptionless.Core/Bootstrapper.cs +++ b/src/Exceptionless.Core/Bootstrapper.cs @@ -1,4 +1,5 @@ using System.Text.Json; +using Elastic.Clients.Elasticsearch; using Exceptionless.Core.Authentication; using Exceptionless.Core.Billing; using Exceptionless.Core.Configuration; @@ -7,7 +8,6 @@ using Exceptionless.Core.Jobs; using Exceptionless.Core.Jobs.WorkItemHandlers; using Exceptionless.Core.Mail; -using Exceptionless.Core.Models; using Exceptionless.Core.Models.WorkItems; using Exceptionless.Core.Pipeline; using Exceptionless.Core.Plugins; @@ -24,7 +24,6 @@ using Exceptionless.Core.Services; using Exceptionless.Core.Utility; using Exceptionless.Core.Validation; -using Exceptionless.Serializer; using Foundatio.Caching; using Foundatio.Extensions.Hosting.Jobs; using Foundatio.Extensions.Hosting.Startup; @@ -43,7 +42,6 @@ using Foundatio.Storage; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; -using DataDictionary = Exceptionless.Core.Models.DataDictionary; using MaintainIndexesJob = Foundatio.Repositories.Elasticsearch.Jobs.MaintainIndexesJob; namespace Exceptionless.Core; @@ -52,27 +50,7 @@ public class Bootstrapper { public static void RegisterServices(IServiceCollection services, AppOptions appOptions) { - // PERF: Work towards getting rid of JSON.NET. - Newtonsoft.Json.JsonConvert.DefaultSettings = () => new Newtonsoft.Json.JsonSerializerSettings - { - DateParseHandling = Newtonsoft.Json.DateParseHandling.DateTimeOffset - }; - - services.AddSingleton(_ => GetJsonContractResolver()); - services.AddSingleton(s => - { - // NOTE: These settings may need to be synced in the Elastic Configuration. - var settings = new Newtonsoft.Json.JsonSerializerSettings - { - MissingMemberHandling = Newtonsoft.Json.MissingMemberHandling.Ignore, - DateParseHandling = Newtonsoft.Json.DateParseHandling.DateTimeOffset, - ContractResolver = s.GetRequiredService() - }; - - settings.AddModelConverters(s.GetRequiredService>()); - return settings; - }); - + // Register System.Text.Json options with Exceptionless defaults (snake_case, null handling) services.AddSingleton(_ => new JsonSerializerOptions().ConfigureExceptionlessDefaults()); services.AddSingleton(s => s.GetRequiredService()); @@ -90,7 +68,7 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO })); services.AddSingleton(); - services.AddSingleton(s => s.GetRequiredService().Client); + services.AddSingleton(s => s.GetRequiredService().Client); services.AddSingleton(s => s.GetRequiredService()); services.AddStartupAction(); @@ -281,13 +259,6 @@ public static void AddHostedJobs(IServiceCollection services, ILoggerFactory log logger.LogWarning("Jobs running in process"); } - public static DynamicTypeContractResolver GetJsonContractResolver() - { - var resolver = new DynamicTypeContractResolver(new LowerCaseUnderscorePropertyNamesContractResolver()); - resolver.UseDefaultResolverFor(typeof(DataDictionary), typeof(SettingsDictionary), typeof(VersionOnePlugin.VersionOneWebHookStack), typeof(VersionOnePlugin.VersionOneWebHookEvent)); - return resolver; - } - private static IQueue CreateQueue(IServiceProvider container, TimeSpan? workItemTimeout = null) where T : class { var loggerFactory = container.GetRequiredService(); diff --git a/src/Exceptionless.Core/Exceptionless.Core.csproj b/src/Exceptionless.Core/Exceptionless.Core.csproj index 47cb9f842e..dea4387014 100644 --- a/src/Exceptionless.Core/Exceptionless.Core.csproj +++ b/src/Exceptionless.Core/Exceptionless.Core.csproj @@ -23,9 +23,7 @@ - - @@ -34,9 +32,7 @@ - - + + diff --git a/src/Exceptionless.Core/Extensions/DataDictionaryExtensions.cs b/src/Exceptionless.Core/Extensions/DataDictionaryExtensions.cs index 6ebf49a9e8..604d975cdc 100644 --- a/src/Exceptionless.Core/Extensions/DataDictionaryExtensions.cs +++ b/src/Exceptionless.Core/Extensions/DataDictionaryExtensions.cs @@ -1,6 +1,12 @@ -using System.Text.Json; +using System.Collections; +using System.Collections.Concurrent; +using System.Reflection; +using System.Text.Json; using System.Text.Json.Nodes; +using System.Text.Json.Serialization.Metadata; using Exceptionless.Core.Models; +using Exceptionless.Core.Serialization; +using Foundatio.Serializer; namespace Exceptionless.Core.Extensions; @@ -9,27 +15,7 @@ public static class DataDictionaryExtensions /// /// Retrieves a typed value from the , deserializing if necessary. /// - /// The target type to deserialize to. - /// The data dictionary containing the value. - /// The key of the value to retrieve. - /// The JSON serializer options to use for deserialization. - /// The deserialized value, or default if deserialization fails. - /// Thrown when the key is not found in the dictionary. - /// - /// This method handles multiple source formats in priority order: - /// - /// Direct type match - returns value directly - /// - extracts root element and deserializes - /// - deserializes using provided options - /// - deserializes using provided options - /// - re-serializes to JSON then deserializes (for ObjectToInferredTypesConverter output) - /// of objects - re-serializes to JSON then deserializes - /// - uses ToObject for Elasticsearch compatibility (data read from Elasticsearch uses JSON.NET) - /// JSON string - parses and deserializes - /// Fallback - attempts type conversion via ToType - /// - /// - public static T? GetValue(this DataDictionary extendedData, string key, JsonSerializerOptions options) + public static T? GetValue(this DataDictionary extendedData, string key, ITextSerializer serializer) { if (!extendedData.TryGetValue(key, out object? data)) throw new KeyNotFoundException($"Key \"{key}\" not found in the dictionary."); @@ -41,73 +27,76 @@ public static class DataDictionaryExtensions if (data is JsonDocument jsonDocument) data = jsonDocument.RootElement; - // JsonElement (from STJ deserialization when ObjectToInferredTypesConverter wasn't used) - if (data is JsonElement jsonElement && - TryDeserialize(jsonElement, options, out T? jsonElementResult)) - { - return jsonElementResult; - } - - // JsonNode (JsonObject/JsonArray/JsonValue) - if (data is JsonNode jsonNode) + if (data is JsonElement jsonElement) { try { - var result = jsonNode.Deserialize(options); - if (result is not null) - return result; + if (typeof(T) == typeof(string)) + { + object? s = jsonElement.ValueKind switch + { + JsonValueKind.String => jsonElement.GetString(), + JsonValueKind.Number => jsonElement.GetRawText(), + JsonValueKind.True => "true", + JsonValueKind.False => "false", + JsonValueKind.Null => null, + _ => jsonElement.GetRawText() + }; + + return (T?)s; + } + + string elementJson = jsonElement.GetRawText(); + return serializer.Deserialize(elementJson); } - catch + catch (Exception ex) when (ex is JsonException or InvalidOperationException or FormatException) { - // Ignored - fall through to next handler + // Ignored - fall through to next handler. } } - // Dictionary from ObjectToInferredTypesConverter - // Re-serialize to JSON then deserialize to target type with proper naming policy - if (data is Dictionary dictionary) + // JsonNode (JsonObject/JsonArray/JsonValue) + if (data is JsonNode jsonNode) { try { - string dictJson = JsonSerializer.Serialize(dictionary, options); - var result = JsonSerializer.Deserialize(dictJson, options); - if (result is not null) - return result; + string jsonString = jsonNode.ToJsonString(); + return serializer.Deserialize(jsonString); } - catch + catch (Exception ex) when (ex is JsonException or InvalidOperationException or FormatException) { - // Ignored - fall through to next handler + // Ignored - fall through to next handler. } } - // List from ObjectToInferredTypesConverter (for array values) - if (data is List list) + if (data is Dictionary dictionary) { try { - string listJson = JsonSerializer.Serialize(list, options); - var result = JsonSerializer.Deserialize(listJson, options); - if (result is not null) - return result; + object? normalizedDictionary = NormalizeValueForType(dictionary, typeof(T)); + string? dictJson = serializer.SerializeToString(normalizedDictionary); + if (dictJson is not null) + return serializer.Deserialize(dictJson); } - catch + catch (Exception ex) when (ex is JsonException or InvalidOperationException or FormatException) { - // Ignored - fall through to next handler + // Ignored - fall through to next handler. } } - // Newtonsoft.Json.Linq.JObject - for Elasticsearch compatibility. - // When data is read from Elasticsearch (which uses JSON.NET via NEST), complex objects - // in DataDictionary are deserialized as JObject. This handler converts them to the target type. - if (data is Newtonsoft.Json.Linq.JObject jObject) + // List from ObjectToInferredTypesConverter (for array values) + if (data is List list) { try { - return jObject.ToObject(); + object? normalizedList = NormalizeValueForType(list, typeof(T)); + string? listJson = serializer.SerializeToString(normalizedList); + if (listJson is not null) + return serializer.Deserialize(listJson); } - catch + catch (Exception ex) when (ex is JsonException or InvalidOperationException or FormatException) { - // Ignored - fall through to next handler + // Ignored - fall through to next handler. } } @@ -116,13 +105,11 @@ public static class DataDictionaryExtensions { try { - var result = JsonSerializer.Deserialize(json, options); - if (result is not null) - return result; + return serializer.Deserialize(json); } - catch + catch (Exception ex) when (ex is JsonException or InvalidOperationException or FormatException or ArgumentException) { - // Ignored - fall through to next handler + // Ignored - fall through to direct type conversion. } } @@ -130,61 +117,132 @@ public static class DataDictionaryExtensions try { if (data != null) - { return data.ToType(); - } } - catch + catch (Exception ex) when (ex is JsonException or InvalidOperationException or FormatException or InvalidCastException or ArgumentException) { - // Ignored + // Ignored - preserve legacy GetValue behavior: failed conversion returns default. } return default; } - private static bool TryDeserialize(JsonElement element, JsonSerializerOptions options, out T? result) + private sealed record JsonPropertyBinding(string JsonName, Type PropertyType); + + private static readonly ConcurrentDictionary> _propertyMaps = new(); + private static readonly JsonSerializerOptions _propertyNameOptions = new JsonSerializerOptions().ConfigureExceptionlessDefaults(); + + private static object? NormalizeValueForType(object? value, Type targetType) { - result = default; + if (value is null) + return null; - try + targetType = Nullable.GetUnderlyingType(targetType) ?? targetType; + + if (value is Dictionary dictionary) + return NormalizeDictionaryForType(dictionary, targetType); + + if (value is List list) { - // Fast-path for common primitives where the element isn't an object/array - // (Deserialize also works for these, but this avoids some edge cases and allocations) - if (typeof(T) == typeof(string)) - { - object? s = element.ValueKind switch - { - JsonValueKind.String => element.GetString(), - JsonValueKind.Number => element.GetRawText(), - JsonValueKind.True => "true", - JsonValueKind.False => "false", - JsonValueKind.Null => null, - _ => element.GetRawText() - }; - - result = (T?)s; - return true; - } + Type? elementType = GetEnumerableElementType(targetType); + if (elementType is null || elementType == typeof(object)) + return list; + + return list.Select(item => NormalizeValueForType(item, elementType)).ToList(); + } + + return value; + } - // General case - var deserialized = element.Deserialize(options); - if (deserialized is not null) + private static object NormalizeDictionaryForType(Dictionary dictionary, Type targetType) + { + if (ShouldPreserveDictionaryKeys(targetType)) + return dictionary; + + Dictionary propertyMap = _propertyMaps.GetOrAdd(targetType, CreatePropertyMap); + if (propertyMap.Count == 0) + return dictionary; + + var normalized = new Dictionary(StringComparer.OrdinalIgnoreCase); + var currentFormatKeys = new Dictionary(StringComparer.OrdinalIgnoreCase); + + foreach ((string key, object? value) in dictionary) + { + if (!propertyMap.TryGetValue(key, out JsonPropertyBinding? property)) { - result = deserialized; - return true; + normalized[key] = value; + continue; } + + object? normalizedValue = NormalizeValueForType(value, property.PropertyType); + bool isCurrentFormatKey = String.Equals(key, property.JsonName, StringComparison.OrdinalIgnoreCase); + + if (normalized.TryGetValue(property.JsonName, out _) && currentFormatKeys.TryGetValue(property.JsonName, out bool existingIsCurrentFormatKey) && existingIsCurrentFormatKey && !isCurrentFormatKey) + continue; + + normalized[property.JsonName] = normalizedValue; + currentFormatKeys[property.JsonName] = isCurrentFormatKey; } - catch + + return normalized; + } + + private static Dictionary CreatePropertyMap(Type targetType) + { + var map = new Dictionary(StringComparer.OrdinalIgnoreCase); + JsonTypeInfo typeInfo = _propertyNameOptions.GetTypeInfo(targetType); + + foreach (JsonPropertyInfo jsonProperty in typeInfo.Properties) { - // Ignored + if (jsonProperty.AttributeProvider is not PropertyInfo property) + continue; + + if (property.GetIndexParameters().Length > 0) + continue; + + string jsonName = jsonProperty.Name; + var binding = new JsonPropertyBinding(jsonName, property.PropertyType); + + map[jsonName] = binding; + map[property.Name] = binding; } - return false; + return map; + } + + private static bool ShouldPreserveDictionaryKeys(Type targetType) + { + targetType = Nullable.GetUnderlyingType(targetType) ?? targetType; + + if (targetType == typeof(object) || targetType == typeof(DataDictionary)) + return true; + + if (typeof(IDictionary).IsAssignableFrom(targetType)) + return true; + + return targetType.GetInterfaces().Any(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(IDictionary<,>)); + } + + private static Type? GetEnumerableElementType(Type targetType) + { + if (targetType == typeof(string) || ShouldPreserveDictionaryKeys(targetType)) + return null; + + if (targetType.IsArray) + return targetType.GetElementType(); + + if (targetType.IsGenericType && targetType.GetGenericTypeDefinition() == typeof(IEnumerable<>)) + return targetType.GetGenericArguments()[0]; + + return targetType.GetInterfaces() + .Where(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(IEnumerable<>)) + .Select(i => i.GetGenericArguments()[0]) + .FirstOrDefault(); } public static void RemoveSensitiveData(this DataDictionary extendedData) { - string[] removeKeys = extendedData.Keys.Where(k => k.StartsWith('-')).ToArray(); + string[] removeKeys = [.. extendedData.Keys.Where(k => k.StartsWith('-'))]; foreach (string key in removeKeys) extendedData.Remove(key); } diff --git a/src/Exceptionless.Core/Extensions/ErrorExtensions.cs b/src/Exceptionless.Core/Extensions/ErrorExtensions.cs index 634f1fa8fc..b07b645cad 100644 --- a/src/Exceptionless.Core/Extensions/ErrorExtensions.cs +++ b/src/Exceptionless.Core/Extensions/ErrorExtensions.cs @@ -1,6 +1,7 @@ -using System.Text.Json; using Exceptionless.Core.Models; using Exceptionless.Core.Models.Data; +using Foundatio.Serializer; +using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Extensions; @@ -46,7 +47,7 @@ public static StackingTarget GetStackingTarget(this Error error) // fallback to default var defaultError = error.GetInnermostError(); var defaultMethod = defaultError.StackTrace?.FirstOrDefault(); - if (defaultMethod is null && error.StackTrace is not null) + if (defaultMethod is null && error.StackTrace is { Count: > 0 }) { defaultMethod = error.StackTrace?.FirstOrDefault(); defaultError = error; @@ -59,9 +60,9 @@ public static StackingTarget GetStackingTarget(this Error error) }; } - public static StackingTarget? GetStackingTarget(this Event ev, JsonSerializerOptions options) + public static StackingTarget? GetStackingTarget(this Event ev, ITextSerializer serializer, ILogger logger) { - var error = ev.GetError(options); + var error = ev.GetError(serializer, logger); return error?.GetStackingTarget(); } diff --git a/src/Exceptionless.Core/Extensions/EventExtensions.cs b/src/Exceptionless.Core/Extensions/EventExtensions.cs index 0797660f87..3581017d5c 100644 --- a/src/Exceptionless.Core/Extensions/EventExtensions.cs +++ b/src/Exceptionless.Core/Extensions/EventExtensions.cs @@ -1,9 +1,8 @@ -using System.Text; -using System.Text.Json; -using Exceptionless.Core.Extensions; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Models.Data; -using Newtonsoft.Json; +using Foundatio.Serializer; +using Microsoft.Extensions.Logging; namespace Exceptionless; @@ -14,18 +13,18 @@ public static bool HasError(this Event ev) return ev.Data is not null && ev.Data.ContainsKey(Event.KnownDataKeys.Error); } - public static Error? GetError(this Event ev, JsonSerializerOptions options) + public static Error? GetError(this Event ev, ITextSerializer serializer, ILogger logger) { if (!ev.HasError()) return null; try { - return ev.Data!.GetValue(Event.KnownDataKeys.Error, options); + return ev.Data!.GetValue(Event.KnownDataKeys.Error, serializer); } - catch (Exception) + catch (Exception ex) { - // Ignored + logger.LogWarning(ex, "Failed to deserialize {DataKey} for event type {EventType}", Event.KnownDataKeys.Error, ev.Type); } return null; @@ -36,52 +35,64 @@ public static bool HasSimpleError(this Event ev) return ev.Data is not null && ev.Data.ContainsKey(Event.KnownDataKeys.SimpleError); } - public static SimpleError? GetSimpleError(this Event ev, JsonSerializerOptions options) + public static void SetError(this Event ev, Error error) + { + ev.Data ??= new DataDictionary(); + ev.Data[Event.KnownDataKeys.Error] = error; + } + + public static void SetSimpleError(this Event ev, SimpleError error) + { + ev.Data ??= new DataDictionary(); + ev.Data[Event.KnownDataKeys.SimpleError] = error; + } + + public static SimpleError? GetSimpleError(this Event ev, ITextSerializer serializer, ILogger logger) { if (!ev.HasSimpleError()) return null; try { - return ev.Data!.GetValue(Event.KnownDataKeys.SimpleError, options); + return ev.Data!.GetValue(Event.KnownDataKeys.SimpleError, serializer); } - catch (Exception) + catch (Exception ex) { - // Ignored + logger.LogWarning(ex, "Failed to deserialize {DataKey} for event type {EventType}", Event.KnownDataKeys.SimpleError, ev.Type); } return null; } - public static RequestInfo? GetRequestInfo(this Event ev, JsonSerializerOptions options) + public static RequestInfo? GetRequestInfo(this Event ev, ITextSerializer serializer, ILogger logger) { if (ev.Data is null || !ev.Data.ContainsKey(Event.KnownDataKeys.RequestInfo)) return null; try { - return ev.Data.GetValue(Event.KnownDataKeys.RequestInfo, options); + return ev.Data.GetValue(Event.KnownDataKeys.RequestInfo, serializer); } - catch (Exception) + catch (Exception ex) { - // Ignored + logger.LogWarning(ex, "Failed to deserialize {DataKey} for event type {EventType}", Event.KnownDataKeys.RequestInfo, ev.Type); } return null; } - public static EnvironmentInfo? GetEnvironmentInfo(this Event ev, JsonSerializerOptions options) + public static EnvironmentInfo? GetEnvironmentInfo(this Event ev, ITextSerializer serializer, ILogger logger) { if (ev.Data is null || !ev.Data.ContainsKey(Event.KnownDataKeys.EnvironmentInfo)) return null; try { - return ev.Data.GetValue(Event.KnownDataKeys.EnvironmentInfo, options); + return ev.Data.GetValue(Event.KnownDataKeys.EnvironmentInfo, serializer); } - catch (Exception) + catch (Exception ex) { - // Ignored + logger.LogWarning(ex, "Failed to deserialize {DataKey} for event type {EventType}", Event.KnownDataKeys.EnvironmentInfo, ev.Type); } return null; @@ -183,18 +194,18 @@ public static void AddRequestInfo(this Event ev, RequestInfo request) /// /// Gets the user info object from extended data. /// - public static UserInfo? GetUserIdentity(this Event ev, JsonSerializerOptions options) + public static UserInfo? GetUserIdentity(this Event ev, ITextSerializer serializer, ILogger logger) { if (ev.Data is null || !ev.Data.ContainsKey(Event.KnownDataKeys.UserInfo)) return null; try { - return ev.Data.GetValue(Event.KnownDataKeys.UserInfo, options); + return ev.Data.GetValue(Event.KnownDataKeys.UserInfo, serializer); } - catch (Exception) + catch (Exception ex) { - // Ignored + logger.LogWarning(ex, "Failed to deserialize {DataKey} for event type {EventType}", Event.KnownDataKeys.UserInfo, ev.Type); } return null; @@ -219,18 +230,18 @@ public static void SetVersion(this Event ev, string? version) ev.Data[Event.KnownDataKeys.Version] = version.Trim(); } - public static SubmissionClient? GetSubmissionClient(this Event ev, JsonSerializerOptions options) + public static SubmissionClient? GetSubmissionClient(this Event ev, ITextSerializer serializer, ILogger logger) { if (ev.Data is null || !ev.Data.ContainsKey(Event.KnownDataKeys.SubmissionClient)) return null; try { - return ev.Data.GetValue(Event.KnownDataKeys.SubmissionClient, options); + return ev.Data.GetValue(Event.KnownDataKeys.SubmissionClient, serializer); } - catch (Exception) + catch (Exception ex) { - // Ignored + logger.LogWarning(ex, "Failed to deserialize {DataKey} for event type {EventType}", Event.KnownDataKeys.SubmissionClient, ev.Type); } return null; @@ -241,18 +252,18 @@ public static bool HasLocation(this Event ev) return ev.Data != null && ev.Data.ContainsKey(Event.KnownDataKeys.Location); } - public static Location? GetLocation(this Event ev, JsonSerializerOptions options) + public static Location? GetLocation(this Event ev, ITextSerializer serializer, ILogger logger) { if (ev.Data is null || !ev.Data.ContainsKey(Event.KnownDataKeys.Location)) return null; try { - return ev.Data.GetValue(Event.KnownDataKeys.Location, options); + return ev.Data.GetValue(Event.KnownDataKeys.Location, serializer); } - catch (Exception) + catch (Exception ex) { - // Ignored + logger.LogWarning(ex, "Failed to deserialize {DataKey} for event type {EventType}", Event.KnownDataKeys.Location, ev.Type); } return null; @@ -301,18 +312,18 @@ public static void SetEnvironmentInfo(this Event ev, EnvironmentInfo? environmen /// /// Gets the stacking info from extended data. /// - public static ManualStackingInfo? GetManualStackingInfo(this Event ev, JsonSerializerOptions options) + public static ManualStackingInfo? GetManualStackingInfo(this Event ev, ITextSerializer serializer, ILogger logger) { if (ev.Data is null || !ev.Data.ContainsKey(Event.KnownDataKeys.ManualStackingInfo)) return null; try { - return ev.Data.GetValue(Event.KnownDataKeys.ManualStackingInfo, options); + return ev.Data.GetValue(Event.KnownDataKeys.ManualStackingInfo, serializer); } - catch (Exception) + catch (Exception ex) { - // Ignored + logger.LogWarning(ex, "Failed to deserialize {DataKey} for event type {EventType}", Event.KnownDataKeys.ManualStackingInfo, ev.Type); } return null; @@ -423,18 +434,18 @@ public static void RemoveUserIdentity(this Event ev) /// /// Gets the user description from extended data. /// - public static UserDescription? GetUserDescription(this Event ev, JsonSerializerOptions options) + public static UserDescription? GetUserDescription(this Event ev, ITextSerializer serializer, ILogger logger) { if (ev.Data is null || !ev.Data.ContainsKey(Event.KnownDataKeys.UserDescription)) return null; try { - return ev.Data.GetValue(Event.KnownDataKeys.UserDescription, options); + return ev.Data.GetValue(Event.KnownDataKeys.UserDescription, serializer); } - catch (Exception) + catch (Exception ex) { - // Ignored + logger.LogWarning(ex, "Failed to deserialize {DataKey} for event type {EventType}", Event.KnownDataKeys.UserDescription, ev.Type); } return null; @@ -469,8 +480,11 @@ public static void SetUserDescription(this Event ev, UserDescription description ev.Data[Event.KnownDataKeys.UserDescription] = description; } - public static byte[] GetBytes(this Event ev, JsonSerializerSettings settings) + /// + /// Serializes an event to UTF-8 JSON bytes using the specified serializer. + /// + public static byte[] GetBytes(this Event ev, ITextSerializer serializer) { - return Encoding.UTF8.GetBytes(ev.ToJson(Formatting.None, settings)); + return serializer.SerializeToBytes(ev); } } diff --git a/src/Exceptionless.Core/Extensions/JsonExtensions.cs b/src/Exceptionless.Core/Extensions/JsonExtensions.cs index 7c3e95e905..3fdda1cf50 100644 --- a/src/Exceptionless.Core/Extensions/JsonExtensions.cs +++ b/src/Exceptionless.Core/Extensions/JsonExtensions.cs @@ -1,155 +1,22 @@ -using System.Collections; -using System.Collections.Concurrent; -using Exceptionless.Core.Models; -using Exceptionless.Core.Models.Data; -using Exceptionless.Core.Reflection; -using Exceptionless.Serializer; -using Microsoft.Extensions.Logging; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; -using Newtonsoft.Json.Serialization; +namespace Exceptionless.Core.Extensions; -namespace Exceptionless.Core.Extensions; - -[System.Runtime.InteropServices.GuidAttribute("4186FC77-AF28-4D51-AAC3-49055DD855A4")] +/// +/// Extension methods for JSON operations using System.Text.Json. +/// For JsonNode/JsonObject operations, see . +/// public static class JsonExtensions { - public static bool IsNullOrEmpty(this JToken target) - { - if (target is null || target.Type == JTokenType.Null) - return true; - - if (target.Type == JTokenType.Object || target.Type == JTokenType.Array) - return !target.HasValues; - - if (target.Type != JTokenType.Property) - return false; - - var value = ((JProperty)target).Value; - if (value.Type == JTokenType.String) - return value.ToString().IsNullOrEmpty(); - - return IsNullOrEmpty(value); - } - - public static bool IsPropertyNullOrEmpty(this JObject target, string name) - { - var property = target.Property(name); - if (property is null) - return true; - - return property.Value.IsNullOrEmpty(); - } - - public static bool RemoveIfNullOrEmpty(this JObject target, string name) - { - if (!target.IsPropertyNullOrEmpty(name)) - return false; - - target.Remove(name); - return true; - } - - public static void RemoveAll(this JObject target, params string[] names) - { - foreach (string name in names) - target.Remove(name); - } - - - public static bool RemoveAllIfNullOrEmpty(this JObject target, params string[] names) - { - if (target.IsNullOrEmpty()) - return false; - - var properties = target.Descendants().OfType().Where(t => names.Contains(t.Name) && t.IsNullOrEmpty()).ToList(); - foreach (var p in properties) - p.Remove(); - - return true; - } - - public static bool Rename(this JObject target, string currentName, string newName) - { - if (String.Equals(currentName, newName)) - return true; - - var property = target.Property(currentName); - if (property is null) - return false; - - property.Replace(new JProperty(newName, property.Value)); - return true; - } - - public static bool RenameOrRemoveIfNullOrEmpty(this JObject target, string currentName, string newName) - { - var property = target.Property(currentName); - if (property is null) - return false; - - bool isNullOrEmpty = target.IsPropertyNullOrEmpty(currentName); - if (isNullOrEmpty) - { - target.Remove(property.Name); - return false; - } - - property.Replace(new JProperty(newName, property.Value)); - return true; - } - - public static void MoveOrRemoveIfNullOrEmpty(this JObject target, JObject source, params string[] names) - { - foreach (string name in names) - { - var property = source.Property(name); - if (property is null) - continue; - - bool isNullOrEmpty = source.IsPropertyNullOrEmpty(name); - source.Remove(property.Name); - - if (isNullOrEmpty) - continue; - - target.Add(name, property.Value); - } - } - - public static bool RenameAll(this JObject target, string currentName, string newName) - { - var properties = target.Descendants().OfType().Where(t => t.Name == currentName).ToList(); - foreach (var p in properties) - { - if (p.Parent is JObject parent) - parent.Rename(currentName, newName); - } - - return true; - } - - public static string? GetPropertyStringValue(this JObject target, string name) - { - if (target.IsPropertyNullOrEmpty(name)) - return null; - - return target.Property(name)?.Value.ToString(); - } - - - public static string? GetPropertyStringValueAndRemove(this JObject target, string name) - { - string? value = target.GetPropertyStringValue(name); - target.Remove(name); - return value; - } - + /// + /// Checks if a string contains JSON content (starts with { or [). + /// public static bool IsJson(this string value) { return value.GetJsonType() != JsonType.None; } + /// + /// Determines the JSON type of a string (Object, Array, or None). + /// public static JsonType GetJsonType(this string value) { if (String.IsNullOrEmpty(value)) @@ -172,120 +39,7 @@ public static JsonType GetJsonType(this string value) return JsonType.None; } - public static string ToJson(this T data, Formatting formatting = Formatting.None, JsonSerializerSettings? settings = null) - { - var serializer = settings is null ? JsonSerializer.CreateDefault() : JsonSerializer.CreateDefault(settings); - serializer.Formatting = formatting; - - using (var sw = new StringWriter()) - { - serializer.Serialize(sw, data, typeof(T)); - return sw.ToString(); - } - } - - public static List? FromJson(this JArray data, JsonSerializerSettings? settings = null) - { - var serializer = settings is null ? JsonSerializer.CreateDefault() : JsonSerializer.CreateDefault(settings); - return data.ToObject>(serializer); - } - - public static T? FromJson(this string data, JsonSerializerSettings? settings = null) - { - var serializer = settings is null ? JsonSerializer.CreateDefault() : JsonSerializer.CreateDefault(settings); - - using (var sw = new StringReader(data)) - using (var sr = new JsonTextReader(sw)) - return serializer.Deserialize(sr); - } - - public static bool TryFromJson(this string data, out T? value, JsonSerializerSettings? settings = null) - { - try - { - value = data.FromJson(settings); - return true; - } - catch (Exception) - { - value = default; - return false; - } - } - - private static readonly ConcurrentDictionary _countAccessors = new(); - public static bool IsValueEmptyCollection(this JsonProperty property, object target) - { - object? value = property.ValueProvider?.GetValue(target); - if (value is null) - return true; - - if (value is ICollection collection) - return collection.Count == 0; - - if (property.PropertyType is null) - return false; - - if (!_countAccessors.ContainsKey(property.PropertyType)) - { - if (typeof(IEnumerable).IsAssignableFrom(property.PropertyType)) - { - var countProperty = property.PropertyType.GetProperty("Count"); - if (countProperty is not null) - _countAccessors.AddOrUpdate(property.PropertyType, LateBinder.GetPropertyAccessor(countProperty)); - else - _countAccessors.AddOrUpdate(property.PropertyType, null); - } - else - { - _countAccessors.AddOrUpdate(property.PropertyType, null); - } - } - - var countAccessor = _countAccessors[property.PropertyType]; - if (countAccessor is null) - return false; - - int count = (int)(countAccessor.GetValue(value) ?? 0); - return count == 0; - } - - public static void AddModelConverters(this JsonSerializerSettings settings, ILogger logger) - { - var knownEventDataTypes = new Dictionary - { - { Event.KnownDataKeys.Error, typeof(Error) }, - { Event.KnownDataKeys.EnvironmentInfo, typeof(EnvironmentInfo) }, - { Event.KnownDataKeys.Location, typeof(Location) }, - { Event.KnownDataKeys.RequestInfo, typeof(RequestInfo) }, - { Event.KnownDataKeys.SimpleError, typeof(SimpleError) }, - { Event.KnownDataKeys.SubmissionClient, typeof(SubmissionClient) }, - { Event.KnownDataKeys.ManualStackingInfo, typeof(ManualStackingInfo) }, - { Event.KnownDataKeys.UserDescription, typeof(UserDescription) }, - { Event.KnownDataKeys.UserInfo, typeof(UserInfo) } - }; - - var knownProjectDataTypes = new Dictionary - { - { Project.KnownDataKeys.SlackToken, typeof(SlackToken) } - }; - settings.Converters.Add(new DataObjectConverter(logger)); - settings.Converters.Add(new DataObjectConverter(logger, knownProjectDataTypes)); - settings.Converters.Add(new DataObjectConverter(logger, knownEventDataTypes)); - settings.Converters.Add(new DataObjectConverter(logger, knownEventDataTypes)); - settings.Converters.Add(new DataObjectConverter(logger)); - settings.Converters.Add(new DataObjectConverter(logger)); - settings.Converters.Add(new DataObjectConverter(logger)); - settings.Converters.Add(new DataObjectConverter(logger)); - settings.Converters.Add(new DataObjectConverter(logger)); - settings.Converters.Add(new DataObjectConverter(logger)); - settings.Converters.Add(new DataObjectConverter(logger)); - settings.Converters.Add(new DataObjectConverter(logger)); - settings.Converters.Add(new DataObjectConverter(logger)); - settings.Converters.Add(new DataObjectConverter(logger)); - settings.Converters.Add(new DataObjectConverter(logger)); - } } public enum JsonType : byte diff --git a/src/Exceptionless.Core/Extensions/JsonNodeExtensions.cs b/src/Exceptionless.Core/Extensions/JsonNodeExtensions.cs new file mode 100644 index 0000000000..cc4fac40bc --- /dev/null +++ b/src/Exceptionless.Core/Extensions/JsonNodeExtensions.cs @@ -0,0 +1,305 @@ +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace Exceptionless.Core.Extensions; + +/// +/// Extension methods for System.Text.Json.Nodes types (JsonNode, JsonObject, JsonArray). +/// Provides helper methods for JSON manipulation during event processing and upgrades. +/// +public static class JsonNodeExtensions +{ + /// + /// Checks if a JsonNode is null or empty (no values for objects/arrays). + /// + public static bool IsNullOrEmpty(this JsonNode? target) + { + if (target is null) + return true; + + if (target is JsonObject obj) + return obj.Count is 0; + + if (target is JsonArray arr) + return arr.Count is 0; + + if (target is JsonValue val) + { + // Check for null value + if (target.GetValueKind() is JsonValueKind.Null) + return true; + + // Check for empty string + if (target.GetValueKind() is JsonValueKind.String) + { + var strValue = val.GetValue(); + return string.IsNullOrEmpty(strValue); + } + } + + return false; + } + + /// + /// Checks if a property in a JsonObject is null or empty. + /// + public static bool IsPropertyNullOrEmpty(this JsonObject target, string name) + { + if (!target.TryGetPropertyValue(name, out var value)) + return true; + + return value.IsNullOrEmpty(); + } + + /// + /// Removes a property if it is null or empty. + /// + /// True if the property was removed, false otherwise. + public static bool RemoveIfNullOrEmpty(this JsonObject target, string name) + { + if (!target.IsPropertyNullOrEmpty(name)) + return false; + + target.Remove(name); + return true; + } + + /// + /// Removes multiple properties from a JsonObject. + /// + public static void RemoveAll(this JsonObject target, params string[] names) + { + foreach (string name in names) + target.Remove(name); + } + + /// + /// Removes all properties with the given names if they are null or empty, recursively. + /// + /// True if any properties were removed, false otherwise. + public static bool RemoveAllIfNullOrEmpty(this JsonObject target, params string[] names) + { + if (target.IsNullOrEmpty()) + return false; + + bool removed = false; + var toRemove = new List<(JsonObject parent, string name)>(); + + foreach (var descendant in target.DescendantsAndSelf().OfType()) + { + foreach (var name in names.Where(n => descendant.IsPropertyNullOrEmpty(n) && descendant.ContainsKey(n))) + { + toRemove.Add((descendant, name)); + } + } + + foreach (var (parent, name) in toRemove) + { + parent.Remove(name); + removed = true; + } + + return removed; + } + + /// + /// Renames a property in a JsonObject while preserving property order. + /// If newName already exists, the renamed value takes precedence (overwrites). + /// + /// True if the property was renamed, false if not found. + public static bool Rename(this JsonObject target, string currentName, string newName) + { + if (string.Equals(currentName, newName)) + return true; + + if (!target.TryGetPropertyValue(currentName, out var value)) + return false; + + // To preserve order, we need to rebuild the object. + // If newName already exists as another property, the renamed value takes precedence. + var properties = target.ToList(); + target.Clear(); + + foreach (var prop in properties) + { + if (prop.Key == currentName) + target.Add(newName, prop.Value); + else if (prop.Key == newName) + continue; // skip: the renamed value takes precedence over existing newName + else + target.Add(prop.Key, prop.Value); + } + + return true; + } + + /// + /// Renames a property or removes it if null or empty, preserving property order. + /// If newName already exists, the renamed value takes precedence (overwrites). + /// + /// True if renamed, false if removed or not found. + public static bool RenameOrRemoveIfNullOrEmpty(this JsonObject target, string currentName, string newName) + { + if (!target.TryGetPropertyValue(currentName, out var value)) + return false; + + bool isNullOrEmpty = value.IsNullOrEmpty(); + if (isNullOrEmpty) + { + target.Remove(currentName); + return false; + } + + // To preserve order, we need to rebuild the object. + // If newName already exists as another property, the renamed value takes precedence. + var properties = target.ToList(); + target.Clear(); + + foreach (var prop in properties) + { + if (prop.Key == currentName) + target.Add(newName, prop.Value); + else if (prop.Key == newName) + continue; // skip: the renamed value takes precedence over existing newName + else + target.Add(prop.Key, prop.Value); + } + + return true; + } + + /// + /// Moves properties from source to target, removing if null or empty. + /// + public static void MoveOrRemoveIfNullOrEmpty(this JsonObject target, JsonObject source, params string[] names) + { + foreach (string name in names.Where(source.ContainsKey)) + { + source.TryGetPropertyValue(name, out var value); + bool isNullOrEmpty = value.IsNullOrEmpty(); + source.Remove(name); + + if (isNullOrEmpty) + continue; + + target.Add(name, value); + } + } + + /// + /// Renames all properties with the given name recursively throughout the JSON tree. + /// + public static bool RenameAll(this JsonObject target, string currentName, string newName) + { + var objectsWithProperty = target.DescendantsAndSelf() + .OfType() + .Where(o => o.ContainsKey(currentName)) + .ToList(); + + foreach (var obj in objectsWithProperty) + { + obj.Rename(currentName, newName); + } + + return objectsWithProperty.Count > 0; + } + + /// + /// Gets a string value from a property, or null if not found or empty. + /// + public static string? GetPropertyStringValue(this JsonObject target, string name) + { + if (target.IsPropertyNullOrEmpty(name)) + return null; + + if (!target.TryGetPropertyValue(name, out var value)) + return null; + + return value?.ToString(); + } + + /// + /// Gets a string value from a property and removes it. + /// + public static string? GetPropertyStringValueAndRemove(this JsonObject target, string name) + { + string? value = target.GetPropertyStringValue(name); + target.Remove(name); + return value; + } + + /// + /// Enumerates all descendant nodes of a JsonNode. + /// + public static IEnumerable Descendants(this JsonNode? node) + { + if (node is null) + yield break; + + if (node is JsonObject obj) + { + foreach (var prop in obj) + { + yield return prop.Value; + if (prop.Value is not null) + { + foreach (var desc in Descendants(prop.Value)) + yield return desc; + } + } + } + else if (node is JsonArray arr) + { + foreach (var item in arr) + { + yield return item; + if (item is not null) + { + foreach (var desc in Descendants(item)) + yield return desc; + } + } + } + } + + /// + /// Enumerates the node itself and all its descendants. + /// + public static IEnumerable DescendantsAndSelf(this JsonNode? node) + { + yield return node; + foreach (var desc in Descendants(node)) + yield return desc; + } + + /// + /// Checks if a JsonNode has any values (for objects: has properties, for arrays: has items). + /// + public static bool HasValues(this JsonNode? node) + { + return !node.IsNullOrEmpty(); + } + + /// + /// Converts a JsonNode to the specified type. + /// + public static T? ToObject(this JsonNode? node, JsonSerializerOptions options) + { + if (node is null) + return default; + + return node.Deserialize(options); + } + + /// + /// Converts a JsonArray to a List of the specified type. + /// + public static List? ToList(this JsonArray? array, JsonSerializerOptions options) + { + if (array is null) + return null; + + return array.Deserialize>(options); + } + +} diff --git a/src/Exceptionless.Core/Extensions/PersistentEventExtensions.cs b/src/Exceptionless.Core/Extensions/PersistentEventExtensions.cs index c0901a893f..6a869d0425 100644 --- a/src/Exceptionless.Core/Extensions/PersistentEventExtensions.cs +++ b/src/Exceptionless.Core/Extensions/PersistentEventExtensions.cs @@ -1,7 +1,8 @@ -using System.Text.Json; using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Models.Data; +using Foundatio.Serializer; +using Microsoft.Extensions.Logging; namespace Exceptionless; @@ -178,7 +179,7 @@ public static bool UpdateSessionStart(this PersistentEvent ev, DateTime lastActi return true; } - public static PersistentEvent ToSessionStartEvent(this PersistentEvent source, JsonSerializerOptions jsonOptions, DateTime? lastActivityUtc = null, bool? isSessionEnd = null, bool hasPremiumFeatures = true, bool includePrivateInformation = true) + public static PersistentEvent ToSessionStartEvent(this PersistentEvent source, ITextSerializer serializer, ILogger logger, DateTime? lastActivityUtc = null, bool? isSessionEnd = null, bool hasPremiumFeatures = true, bool includePrivateInformation = true) { var startEvent = new PersistentEvent { @@ -194,11 +195,11 @@ public static PersistentEvent ToSessionStartEvent(this PersistentEvent source, J if (sessionId is not null) startEvent.SetSessionId(sessionId); if (includePrivateInformation) - startEvent.SetUserIdentity(source.GetUserIdentity(jsonOptions)); - startEvent.SetLocation(source.GetLocation(jsonOptions)); + startEvent.SetUserIdentity(source.GetUserIdentity(serializer, logger)); + startEvent.SetLocation(source.GetLocation(serializer, logger)); startEvent.SetVersion(source.GetVersion()); - var ei = source.GetEnvironmentInfo(jsonOptions); + var ei = source.GetEnvironmentInfo(serializer, logger); if (ei is not null) { startEvent.SetEnvironmentInfo(new EnvironmentInfo @@ -219,7 +220,7 @@ public static PersistentEvent ToSessionStartEvent(this PersistentEvent source, J }); } - var ri = source.GetRequestInfo(jsonOptions); + var ri = source.GetRequestInfo(serializer, logger); if (ri is not null) { startEvent.AddRequestInfo(new RequestInfo @@ -245,19 +246,19 @@ public static PersistentEvent ToSessionStartEvent(this PersistentEvent source, J return startEvent; } - public static IEnumerable GetIpAddresses(this PersistentEvent ev, JsonSerializerOptions jsonOptions) + public static IEnumerable GetIpAddresses(this PersistentEvent ev, ITextSerializer serializer, ILogger logger) { if (!String.IsNullOrEmpty(ev.Geo) && (ev.Geo.Contains('.') || ev.Geo.Contains(':'))) yield return ev.Geo.Trim(); - var ri = ev.GetRequestInfo(jsonOptions); + var ri = ev.GetRequestInfo(serializer, logger); if (!String.IsNullOrEmpty(ri?.ClientIpAddress)) { foreach (string ip in ri.ClientIpAddress.Split(_commaSeparator, StringSplitOptions.RemoveEmptyEntries)) yield return ip.Trim(); } - var ei = ev.GetEnvironmentInfo(jsonOptions); + var ei = ev.GetEnvironmentInfo(serializer, logger); if (!String.IsNullOrEmpty(ei?.IpAddress)) { foreach (string ip in ei.IpAddress.Split(_commaSeparator, StringSplitOptions.RemoveEmptyEntries)) diff --git a/src/Exceptionless.Core/Extensions/ProjectExtensions.cs b/src/Exceptionless.Core/Extensions/ProjectExtensions.cs index 3f14a0fb30..e666e83832 100644 --- a/src/Exceptionless.Core/Extensions/ProjectExtensions.cs +++ b/src/Exceptionless.Core/Extensions/ProjectExtensions.cs @@ -1,6 +1,9 @@ using System.Text; +using System.Text.Json; using Exceptionless.Core.Models; using Exceptionless.DateTimeExtensions; +using Foundatio.Serializer; +using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Extensions; @@ -48,9 +51,21 @@ public static string BuildFilter(this IList projects) /// /// Gets the slack token from extended data. /// - public static SlackToken? GetSlackToken(this Project project) + public static SlackToken? GetSlackToken(this Project project, ITextSerializer serializer, ILogger? logger = null) { - return project.Data is not null && project.Data.TryGetValue(Project.KnownDataKeys.SlackToken, out object? value) ? value as SlackToken : null; + if (project.Data is null || !project.Data.TryGetValue(Project.KnownDataKeys.SlackToken, out _)) + return null; + + try + { + return project.Data.GetValue(Project.KnownDataKeys.SlackToken, serializer); + } + catch (Exception ex) when (ex is JsonException or InvalidOperationException or FormatException) + { + logger?.LogWarning(ex, "Failed to deserialize SlackToken for project {ProjectId}", project.Id); + } + + return null; } public static bool HasHourlyUsage(this Project project, DateTime date) diff --git a/src/Exceptionless.Core/Extensions/RequestInfoExtensions.cs b/src/Exceptionless.Core/Extensions/RequestInfoExtensions.cs index 1f992b7a58..8019f6a9ae 100644 --- a/src/Exceptionless.Core/Extensions/RequestInfoExtensions.cs +++ b/src/Exceptionless.Core/Extensions/RequestInfoExtensions.cs @@ -1,35 +1,34 @@ using System.Text; using Exceptionless.Core.Models.Data; -using Newtonsoft.Json; +using Foundatio.Serializer; namespace Exceptionless.Core.Extensions; public static class RequestInfoExtensions { - public static RequestInfo ApplyDataExclusions(this RequestInfo request, IList exclusions, int maxLength = 1000) + public static RequestInfo ApplyDataExclusions(this RequestInfo request, ITextSerializer serializer, IList exclusions, int maxLength = 1000) { request.Cookies = ApplyExclusions(request.Cookies, exclusions, maxLength); request.QueryString = ApplyExclusions(request.QueryString, exclusions, maxLength); - request.PostData = ApplyPostDataExclusions(request.PostData, exclusions, maxLength); + request.PostData = ApplyPostDataExclusions(request.PostData, serializer, exclusions, maxLength); return request; } - private static object? ApplyPostDataExclusions(object? data, IEnumerable exclusions, int maxLength) + private static object? ApplyPostDataExclusions(object? data, ITextSerializer serializer, IEnumerable exclusions, int maxLength) { if (data is null) return null; var dictionary = data as Dictionary; - if (dictionary is null && data is string) + if (dictionary is null && data is string json) { - string json = (string)data; if (!json.IsJson()) return data; try { - dictionary = JsonConvert.DeserializeObject>(json); + dictionary = serializer.Deserialize>(json); } catch (Exception) { } } diff --git a/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs b/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs index 6d74dee791..37a67ae24a 100644 --- a/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs +++ b/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs @@ -1,4 +1,8 @@ -using Exceptionless.Core.Models; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Aggregations; +using Elastic.Clients.Elasticsearch.Core.ReindexRethrottle; +using Elastic.Clients.Elasticsearch.QueryDsl; +using Exceptionless.Core.Models; using Exceptionless.Core.Repositories; using Exceptionless.Core.Repositories.Configuration; using Foundatio.Caching; @@ -6,12 +10,12 @@ using Foundatio.Lock; using Foundatio.Repositories; using Foundatio.Repositories.Elasticsearch.Extensions; +using Foundatio.Repositories.Elasticsearch.Utility; using Foundatio.Repositories.Extensions; using Foundatio.Repositories.Models; using Foundatio.Resilience; using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Logging; -using Nest; using LogLevel = Microsoft.Extensions.Logging.LogLevel; namespace Exceptionless.Core.Jobs; @@ -20,14 +24,17 @@ namespace Exceptionless.Core.Jobs; public class CleanupOrphanedDataJob : JobWithLockBase, IHealthCheck { private readonly ExceptionlessElasticConfiguration _config; - private readonly IElasticClient _elasticClient; + private readonly ElasticsearchClient _elasticClient; private readonly IStackRepository _stackRepository; + private readonly IProjectRepository _projectRepository; + private readonly IOrganizationRepository _organizationRepository; private readonly IEventRepository _eventRepository; private readonly ICacheClient _cacheClient; private readonly ILockProvider _lockProvider; private DateTime? _lastRun; public CleanupOrphanedDataJob(ExceptionlessElasticConfiguration config, IStackRepository stackRepository, + IProjectRepository projectRepository, IOrganizationRepository organizationRepository, IEventRepository eventRepository, ICacheClient cacheClient, ILockProvider lockProvider, TimeProvider timeProvider, IResiliencePolicyProvider resiliencePolicyProvider, @@ -37,6 +44,8 @@ ILoggerFactory loggerFactory _config = config; _elasticClient = config.Client; _stackRepository = stackRepository; + _projectRepository = projectRepository; + _organizationRepository = organizationRepository; _eventRepository = eventRepository; _cacheClient = cacheClient; _lockProvider = lockProvider; @@ -61,10 +70,12 @@ protected override async Task RunInternalAsync(JobContext context) public async Task DeleteOrphanedEventsByStackAsync(JobContext context) { // get approximate number of unique stack ids - var stackCardinality = await _elasticClient.SearchAsync(s => s.Size(0).Aggregations(a => a - .Cardinality("cardinality_stack_id", c => c.Field(f => f.StackId).PrecisionThreshold(40000)))); + var stackCardinality = await _elasticClient.SearchAsync(s => s + .Indices(GetEventIndexPattern()) + .Size(0) + .AddAggregation("cardinality_stack_id", a => a.Cardinality(c => c.Field(f => f.StackId).PrecisionThreshold(40000)))); - double? uniqueStackIdCount = stackCardinality.Aggregations.Cardinality("cardinality_stack_id")?.Value; + double? uniqueStackIdCount = stackCardinality.Aggregations?.GetCardinality("cardinality_stack_id")?.Value; if (!uniqueStackIdCount.HasValue || uniqueStackIdCount.Value <= 0) return; @@ -79,18 +90,20 @@ public async Task DeleteOrphanedEventsByStackAsync(JobContext context) { await RenewLockAsync(context); - var stackIdTerms = await _elasticClient.SearchAsync(s => s.Size(0).Aggregations(a => a - .Terms("terms_stack_id", c => c.Field(f => f.StackId).Include(batchNumber, buckets).Size(batchSize * 2)))); + var stackIdTerms = await _elasticClient.SearchAsync(s => s + .Indices(GetEventIndexPattern()) + .Size(0) + .AddAggregation("terms_stack_id", a => a.Terms(c => c.Field(f => f.StackId).Include(new TermsInclude(batchNumber, buckets)).Size(batchSize * 2)))); - string[] stackIds = stackIdTerms.Aggregations.Terms("terms_stack_id").Buckets.Select(b => b.Key).ToArray(); + string[] stackIds = stackIdTerms.Aggregations?.GetStringTerms("terms_stack_id")?.Buckets.Select(b => b.Key.ToString()!).ToArray() ?? []; if (stackIds.Length == 0) continue; totalStackIds += stackIds.Length; - var stacks = await _elasticClient.MultiGetAsync(r => r.SourceEnabled(false).GetMany(stackIds)); - string[] missingStackIds = stacks.Hits.Where(h => !h.Found).Select(h => h.Id).ToArray(); - + var stacks = await _stackRepository.GetByIdsAsync(stackIds, o => o.ImmediateConsistency()); + var foundStackIds = stacks.Select(stack => stack.Id).ToHashSet(StringComparer.OrdinalIgnoreCase); + string[] missingStackIds = stackIds.Where(stackId => !foundStackIds.Contains(stackId)).ToArray(); if (missingStackIds.Length == 0) { @@ -100,7 +113,9 @@ public async Task DeleteOrphanedEventsByStackAsync(JobContext context) totalOrphanedEventCount += missingStackIds.Length; _logger.LogInformation("{BatchNumber}/{BatchCount}: Found {OrphanedEventCount} orphaned events from missing stacks {MissingStackIds} out of {StackIdCount}", batchNumber, buckets, missingStackIds.Length, missingStackIds, stackIds.Length); - await _elasticClient.DeleteByQueryAsync(r => r.Query(q => q.Terms(t => t.Field(f => f.StackId).Terms(missingStackIds)))); + await _elasticClient.DeleteByQueryAsync(r => r + .Indices(GetEventIndexPattern()) + .Query(q => q.Terms(t => t.Field(f => f.StackId).Terms(new TermsQueryField(missingStackIds.Select(FieldValueHelper.ToFieldValue).ToList()))))); } _logger.LogInformation("Found {OrphanedEventCount} orphaned events from missing stacks out of {StackIdCount}", totalOrphanedEventCount, totalStackIds); @@ -109,10 +124,12 @@ public async Task DeleteOrphanedEventsByStackAsync(JobContext context) public async Task DeleteOrphanedEventsByProjectAsync(JobContext context) { // get approximate number of unique project ids - var projectCardinality = await _elasticClient.SearchAsync(s => s.Size(0).Aggregations(a => a - .Cardinality("cardinality_project_id", c => c.Field(f => f.ProjectId).PrecisionThreshold(40000)))); + var projectCardinality = await _elasticClient.SearchAsync(s => s + .Indices(GetEventIndexPattern()) + .Size(0) + .AddAggregation("cardinality_project_id", a => a.Cardinality(c => c.Field(f => f.ProjectId).PrecisionThreshold(40000)))); - double? uniqueProjectIdCount = projectCardinality.Aggregations.Cardinality("cardinality_project_id")?.Value; + double? uniqueProjectIdCount = projectCardinality.Aggregations?.GetCardinality("cardinality_project_id")?.Value; if (!uniqueProjectIdCount.HasValue || uniqueProjectIdCount.Value <= 0) return; @@ -127,17 +144,20 @@ public async Task DeleteOrphanedEventsByProjectAsync(JobContext context) { await RenewLockAsync(context); - var projectIdTerms = await _elasticClient.SearchAsync(s => s.Size(0).Aggregations(a => a - .Terms("terms_project_id", c => c.Field(f => f.ProjectId).Include(batchNumber, buckets).Size(batchSize * 2)))); + var projectIdTerms = await _elasticClient.SearchAsync(s => s + .Indices(GetEventIndexPattern()) + .Size(0) + .AddAggregation("terms_project_id", a => a.Terms(c => c.Field(f => f.ProjectId).Include(new TermsInclude(batchNumber, buckets)).Size(batchSize * 2)))); - string[] projectIds = projectIdTerms.Aggregations.Terms("terms_project_id").Buckets.Select(b => b.Key).ToArray(); + string[] projectIds = projectIdTerms.Aggregations?.GetStringTerms("terms_project_id")?.Buckets.Select(b => b.Key.ToString()!).ToArray() ?? []; if (projectIds.Length == 0) continue; totalProjectIds += projectIds.Length; - var projects = await _elasticClient.MultiGetAsync(r => r.SourceEnabled(false).GetMany(projectIds)); - string[] missingProjectIds = projects.Hits.Where(h => !h.Found).Select(h => h.Id).ToArray(); + var projects = await _projectRepository.GetByIdsAsync(projectIds, o => o.ImmediateConsistency()); + var foundProjectIds = projects.Select(project => project.Id).ToHashSet(StringComparer.OrdinalIgnoreCase); + string[] missingProjectIds = projectIds.Where(projectId => !foundProjectIds.Contains(projectId)).ToArray(); if (missingProjectIds.Length == 0) { @@ -145,8 +165,11 @@ public async Task DeleteOrphanedEventsByProjectAsync(JobContext context) continue; } + totalOrphanedEventCount += missingProjectIds.Length; _logger.LogInformation("{BatchNumber}/{BatchCount}: Found {OrphanedEventCount} orphaned events from missing projects {MissingProjectIds} out of {ProjectIdCount}", batchNumber, buckets, missingProjectIds.Length, missingProjectIds, projectIds.Length); - await _elasticClient.DeleteByQueryAsync(r => r.Query(q => q.Terms(t => t.Field(f => f.ProjectId).Terms(missingProjectIds)))); + await _elasticClient.DeleteByQueryAsync(r => r + .Indices(GetEventIndexPattern()) + .Query(q => q.Terms(t => t.Field(f => f.ProjectId).Terms(new TermsQueryField(missingProjectIds.Select(FieldValueHelper.ToFieldValue).ToList()))))); } _logger.LogInformation("Found {OrphanedEventCount} orphaned events from missing projects out of {ProjectIdCount}", totalOrphanedEventCount, totalProjectIds); @@ -155,10 +178,12 @@ public async Task DeleteOrphanedEventsByProjectAsync(JobContext context) public async Task DeleteOrphanedEventsByOrganizationAsync(JobContext context) { // get approximate number of unique organization ids - var organizationCardinality = await _elasticClient.SearchAsync(s => s.Size(0).Aggregations(a => a - .Cardinality("cardinality_organization_id", c => c.Field(f => f.OrganizationId).PrecisionThreshold(40000)))); + var organizationCardinality = await _elasticClient.SearchAsync(s => s + .Indices(GetEventIndexPattern()) + .Size(0) + .AddAggregation("cardinality_organization_id", a => a.Cardinality(c => c.Field(f => f.OrganizationId).PrecisionThreshold(40000)))); - double? uniqueOrganizationIdCount = organizationCardinality.Aggregations.Cardinality("cardinality_organization_id")?.Value; + double? uniqueOrganizationIdCount = organizationCardinality.Aggregations?.GetCardinality("cardinality_organization_id")?.Value; if (!uniqueOrganizationIdCount.HasValue || uniqueOrganizationIdCount.Value <= 0) return; @@ -173,17 +198,20 @@ public async Task DeleteOrphanedEventsByOrganizationAsync(JobContext context) { await RenewLockAsync(context); - var organizationIdTerms = await _elasticClient.SearchAsync(s => s.Size(0).Aggregations(a => a - .Terms("terms_organization_id", c => c.Field(f => f.OrganizationId).Include(batchNumber, buckets).Size(batchSize * 2)))); + var organizationIdTerms = await _elasticClient.SearchAsync(s => s + .Indices(GetEventIndexPattern()) + .Size(0) + .AddAggregation("terms_organization_id", a => a.Terms(c => c.Field(f => f.OrganizationId).Include(new TermsInclude(batchNumber, buckets)).Size(batchSize * 2)))); - string[] organizationIds = organizationIdTerms.Aggregations.Terms("terms_organization_id").Buckets.Select(b => b.Key).ToArray(); + string[] organizationIds = organizationIdTerms.Aggregations?.GetStringTerms("terms_organization_id")?.Buckets.Select(b => b.Key.ToString()!).ToArray() ?? []; if (organizationIds.Length == 0) continue; totalOrganizationIds += organizationIds.Length; - var organizations = await _elasticClient.MultiGetAsync(r => r.SourceEnabled(false).GetMany(organizationIds)); - string[] missingOrganizationIds = organizations.Hits.Where(h => !h.Found).Select(h => h.Id).ToArray(); + var organizations = await _organizationRepository.GetByIdsAsync(organizationIds, o => o.ImmediateConsistency()); + var foundOrganizationIds = organizations.Select(organization => organization.Id).ToHashSet(StringComparer.OrdinalIgnoreCase); + string[] missingOrganizationIds = organizationIds.Where(organizationId => !foundOrganizationIds.Contains(organizationId)).ToArray(); if (missingOrganizationIds.Length == 0) { @@ -191,8 +219,11 @@ public async Task DeleteOrphanedEventsByOrganizationAsync(JobContext context) continue; } + totalOrphanedEventCount += missingOrganizationIds.Length; _logger.LogInformation("{BatchNumber}/{BatchCount}: Found {OrphanedEventCount} orphaned events from missing organizations {MissingOrganizationIds} out of {OrganizationIdCount}", batchNumber, buckets, missingOrganizationIds.Length, missingOrganizationIds, organizationIds.Length); - await _elasticClient.DeleteByQueryAsync(r => r.Query(q => q.Terms(t => t.Field(f => f.OrganizationId).Terms(missingOrganizationIds)))); + await _elasticClient.DeleteByQueryAsync(r => r + .Indices(GetEventIndexPattern()) + .Query(q => q.Terms(t => t.Field(f => f.OrganizationId).Terms(new TermsQueryField(missingOrganizationIds.Select(FieldValueHelper.ToFieldValue).ToList()))))); } _logger.LogInformation("Found {OrphanedEventCount} orphaned events from missing organizations out of {OrganizationIdCount}", totalOrphanedEventCount, totalOrganizationIds); @@ -203,12 +234,13 @@ public async Task FixDuplicateStacks(JobContext context) _logger.LogInformation("Getting duplicate stacks"); var duplicateStackAgg = await _elasticClient.SearchAsync(q => q - .QueryOnQueryString("is_deleted:false") + .Indices(_config.Stacks.VersionedName) + .Query(q => q.QueryString(qs => qs.Query("is_deleted:false"))) .Size(0) - .Aggregations(a => a.Terms("stacks", t => t.Field(f => f.DuplicateSignature).MinimumDocumentCount(2).Size(10000)))); + .AddAggregation("stacks", a => a.Terms(t => t.Field(f => f.DuplicateSignature).MinDocCount(2).Size(10000)))); _logger.LogRequest(duplicateStackAgg, LogLevel.Trace); - var buckets = duplicateStackAgg.Aggregations.Terms("stacks")?.Buckets ?? new List>(); + var buckets = duplicateStackAgg.Aggregations?.GetStringTerms("stacks")?.Buckets.ToList() ?? []; int total = buckets.Count; int processed = 0; int error = 0; @@ -227,16 +259,16 @@ public async Task FixDuplicateStacks(JobContext context) string? signature = null; try { - string[] parts = duplicateSignature.Key.Split(':'); + string[] parts = duplicateSignature.Key.ToString().Split(':'); if (parts.Length != 2) { - _logger.LogError("Error parsing duplicate signature {DuplicateSignature}", duplicateSignature.Key); + _logger.LogError("Error parsing duplicate signature {DuplicateSignature}", duplicateSignature.Key.ToString()); continue; } projectId = parts[0]; signature = parts[1]; - var stacks = await _stackRepository.FindAsync(q => q.Project(projectId).FilterExpression($"signature_hash:{signature}")); + var stacks = await _stackRepository.FindAsync(q => q.Project(projectId).FieldEquals(s => s.SignatureHash, signature)); if (stacks.Documents.Count < 2) { _logger.LogError("Did not find multiple stacks with signature {SignatureHash} and project {ProjectId}", signature, projectId); @@ -282,11 +314,12 @@ public async Task FixDuplicateStacks(JobContext context) if (shouldUpdateEvents) { var response = await _elasticClient.UpdateByQueryAsync(u => u + .Indices(GetEventIndexPattern()) .Query(q => q.Bool(b => b.Must(m => m - .Terms(t => t.Field(f => f.StackId).Terms(duplicateStacks.Select(s => s.Id))) + .Terms(t => t.Field(f => f.StackId).Terms(new TermsQueryField(duplicateStacks.Select(s => FieldValueHelper.ToFieldValue(s.Id)).ToList()))) ))) - .Script(s => s.Source($"ctx._source.stack_id = '{targetStack.Id}'").Lang(ScriptLang.Painless)) - .Conflicts(Elasticsearch.Net.Conflicts.Proceed) + .Script(s => s.Source($"ctx._source.stack_id = '{targetStack.Id}'").Lang(ScriptLanguage.Painless)) + .Conflicts(Conflicts.Proceed) .WaitForCompletion(false)); _logger.LogRequest(response, LogLevel.Trace); @@ -297,22 +330,22 @@ public async Task FixDuplicateStacks(JobContext context) do { attempts++; - var taskStatus = await _elasticClient.Tasks.GetTaskAsync(taskId); - var status = taskStatus.Task.Status; + var taskStatus = await _elasticClient.Tasks.GetAsync(taskId!.FullyQualifiedId); + var status = taskStatus.Task.Status as ReindexStatus; if (taskStatus.Completed) { // TODO: need to check to see if the task failed or completed successfully. Throw if it failed. if (_timeProvider.GetUtcNow().UtcDateTime.Subtract(taskStartedTime) > TimeSpan.FromSeconds(30)) - _logger.LogInformation("Script operation task ({TaskId}) completed: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total); + _logger.LogInformation("Script operation task ({TaskId}) completed: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status?.Created, status?.Updated, status?.Deleted, status?.VersionConflicts, status?.Total); - affectedRecords += status.Created + status.Updated + status.Deleted; + affectedRecords += (status?.Created ?? 0) + (status?.Updated ?? 0) + (status?.Deleted ?? 0); break; } if (_timeProvider.GetUtcNow().UtcDateTime.Subtract(taskStartedTime) > TimeSpan.FromSeconds(30)) { await RenewLockAsync(context); - _logger.LogInformation("Checking script operation task ({TaskId}) status: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total); + _logger.LogInformation("Checking script operation task ({TaskId}) status: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status?.Created, status?.Updated, status?.Deleted, status?.VersionConflicts, status?.Total); } var delay = TimeSpan.FromMilliseconds(50); @@ -323,7 +356,7 @@ public async Task FixDuplicateStacks(JobContext context) else if (attempts > 5) delay = TimeSpan.FromMilliseconds(250); - await Task.Delay(delay, _timeProvider); + await Task.Delay(delay, _timeProvider, context.CancellationToken); } while (true); _logger.LogInformation("Migrated stack events: Target={TargetId} Events={UpdatedEvents} Dupes={DuplicateIds}", targetStack.Id, affectedRecords, duplicateStacks.Select(s => s.Id)); @@ -347,12 +380,13 @@ public async Task FixDuplicateStacks(JobContext context) await _elasticClient.Indices.RefreshAsync(_config.Stacks.VersionedName); duplicateStackAgg = await _elasticClient.SearchAsync(q => q - .QueryOnQueryString("is_deleted:false") + .Indices(_config.Stacks.VersionedName) + .Query(q => q.QueryString(qs => qs.Query("is_deleted:false"))) .Size(0) - .Aggregations(a => a.Terms("stacks", t => t.Field(f => f.DuplicateSignature).MinimumDocumentCount(2).Size(10000)))); + .AddAggregation("stacks", a => a.Terms(t => t.Field(f => f.DuplicateSignature).MinDocCount(2).Size(10000)))); _logger.LogRequest(duplicateStackAgg, LogLevel.Trace); - buckets = duplicateStackAgg.Aggregations.Terms("stacks").Buckets; + buckets = duplicateStackAgg.Aggregations?.GetStringTerms("stacks")?.Buckets.ToList() ?? []; total += buckets.Count; batch++; @@ -367,6 +401,11 @@ private Task RenewLockAsync(JobContext context) return context.RenewLockAsync(); } + private string GetEventIndexPattern() + { + return $"{_config.Events.VersionedName}-*"; + } + public Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) { if (!_lastRun.HasValue) diff --git a/src/Exceptionless.Core/Jobs/CloseInactiveSessionsJob.cs b/src/Exceptionless.Core/Jobs/CloseInactiveSessionsJob.cs index 1729857e0e..30d607eaab 100644 --- a/src/Exceptionless.Core/Jobs/CloseInactiveSessionsJob.cs +++ b/src/Exceptionless.Core/Jobs/CloseInactiveSessionsJob.cs @@ -1,5 +1,4 @@ using System.Diagnostics; -using System.Text.Json; using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Repositories; @@ -9,6 +8,7 @@ using Foundatio.Lock; using Foundatio.Repositories; using Foundatio.Resilience; +using Foundatio.Serializer; using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Logging; @@ -20,11 +20,11 @@ public class CloseInactiveSessionsJob : JobWithLockBase, IHealthCheck private readonly IEventRepository _eventRepository; private readonly ICacheClient _cache; private readonly ILockProvider _lockProvider; - private readonly JsonSerializerOptions _jsonOptions; + private readonly ITextSerializer _serializer; private DateTime? _lastActivity; public CloseInactiveSessionsJob(IEventRepository eventRepository, ICacheClient cacheClient, - JsonSerializerOptions jsonOptions, + ITextSerializer serializer, TimeProvider timeProvider, IResiliencePolicyProvider resiliencePolicyProvider, ILoggerFactory loggerFactory @@ -33,7 +33,7 @@ ILoggerFactory loggerFactory _eventRepository = eventRepository; _cache = cacheClient; _lockProvider = new ThrottlingLockProvider(cacheClient, 1, TimeSpan.FromMinutes(1), timeProvider, resiliencePolicyProvider, loggerFactory); - _jsonOptions = jsonOptions; + _serializer = serializer; } protected override Task GetLockAsync(CancellationToken cancellationToken = default) @@ -130,7 +130,7 @@ protected override async Task RunInternalAsync(JobContext context) allHeartbeatKeys.Add(sessionIdKey); } - var user = session.GetUserIdentity(_jsonOptions); + var user = session.GetUserIdentity(_serializer, _logger); if (!String.IsNullOrWhiteSpace(user?.Identity)) { userIdentityKey = $"Project:{session.ProjectId}:heartbeat:{user.Identity.ToSHA1()}"; diff --git a/src/Exceptionless.Core/Jobs/Elastic/DataMigrationJob.cs b/src/Exceptionless.Core/Jobs/Elastic/DataMigrationJob.cs index 41a670598d..2fccf85fa7 100644 --- a/src/Exceptionless.Core/Jobs/Elastic/DataMigrationJob.cs +++ b/src/Exceptionless.Core/Jobs/Elastic/DataMigrationJob.cs @@ -1,4 +1,7 @@ -using Elasticsearch.Net; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Core.Reindex; +using Elastic.Clients.Elasticsearch.Core.ReindexRethrottle; +using Elastic.Clients.Elasticsearch.Tasks; using Exceptionless.Core.Repositories.Configuration; using Exceptionless.DateTimeExtensions; using Foundatio.Caching; @@ -8,7 +11,6 @@ using Foundatio.Repositories.Elasticsearch.Extensions; using Foundatio.Resilience; using Microsoft.Extensions.Logging; -using Nest; namespace Exceptionless.Core.Jobs.Elastic; @@ -98,29 +100,33 @@ protected override async Task RunInternalAsync(JobContext context) else if (dequeuedWorkItem.Attempts >= 2) batchSize = 250; - var response = await client.ReindexOnServerAsync(r => r + var response = await client.ReindexAsync(r => r .Source(s => s .Remote(ConfigureRemoteElasticSource) - .Index(dequeuedWorkItem.SourceIndex) + .Indices(dequeuedWorkItem.SourceIndex) .Size(batchSize) - .Query(q => + .Query(q => { - var container = q.Term("_type", dequeuedWorkItem.SourceIndexType); if (!String.IsNullOrEmpty(dequeuedWorkItem.DateField)) - container &= q.DateRange(d => d.Field(dequeuedWorkItem.DateField).GreaterThanOrEquals(cutOffDate)); - - return container; + { + q.Bool(b => b.Must( + m => m.Term(t => t.Field("_type").Value(dequeuedWorkItem.SourceIndexType)), + m => m.Range(r => r.Date(d => d.Field(dequeuedWorkItem.DateField!).Gte(cutOffDate))) + )); + } + else + { + q.Term(t => t.Field("_type").Value(dequeuedWorkItem.SourceIndexType)); + } })) - .Destination(d => d + .Dest(d => d .Index(dequeuedWorkItem.TargetIndex)) .Conflicts(Conflicts.Proceed) .WaitForCompletion(false) .Script(s => { if (!String.IsNullOrEmpty(dequeuedWorkItem.Script)) - return s.Source(dequeuedWorkItem.Script); - - return null; + s.Source(dequeuedWorkItem.Script); })); dequeuedWorkItem.Attempts += 1; @@ -135,26 +141,26 @@ protected override async Task RunInternalAsync(JobContext context) double highestProgress = 0; foreach (var workItem in workingTasks.ToArray()) { - var taskStatus = await client.Tasks.GetTaskAsync(workItem.TaskId, t => t.WaitForCompletion(false)); + var taskStatus = await client.Tasks.GetAsync(workItem.TaskId!.FullyQualifiedId, t => t.WaitForCompletion(false)); _logger.LogRequest(taskStatus); - var status = taskStatus?.Task?.Status; + var status = taskStatus?.Task?.Status as ReindexStatus; if (taskStatus?.Task is null || status is null) { - _logger.LogWarning(taskStatus?.OriginalException, "Error getting task status for {TargetIndex} {TaskId}: {Message}", workItem.TargetIndex, workItem.TaskId, taskStatus?.GetErrorMessage()); - if (taskStatus?.ServerError?.Status == 429) + _logger.LogWarning(taskStatus?.ApiCallDetails?.OriginalException, "Error getting task status for {TargetIndex} {TaskId}: {Message}", workItem.TargetIndex, workItem.TaskId, taskStatus?.GetErrorMessage()); + if (taskStatus?.ElasticsearchServerError?.Status == 429) await Task.Delay(TimeSpan.FromSeconds(1), _timeProvider); continue; } - var duration = TimeSpan.FromMilliseconds(taskStatus.Task.RunningTimeInNanoseconds * 0.000001); + var duration = taskStatus.Task.RunningTimeInNanos; double progress = status.Total > 0 ? (status.Created + status.Updated + status.Deleted + status.VersionConflicts * 1.0) / status.Total : 0; highestProgress = Math.Max(highestProgress, progress); - if (!taskStatus.IsValid) + if (!taskStatus.IsValidResponse) { - _logger.LogWarning(taskStatus.OriginalException, "Error getting task status for {TargetIndex} ({TaskId}): {Message}", workItem.TargetIndex, workItem.TaskId, taskStatus.GetErrorMessage()); + _logger.LogWarning(taskStatus.ApiCallDetails?.OriginalException, "Error getting task status for {TargetIndex} ({TaskId}): {Message}", workItem.TargetIndex, workItem.TaskId, taskStatus.GetErrorMessage()); workItem.ConsecutiveStatusErrors++; if (taskStatus.Completed || workItem.ConsecutiveStatusErrors > 5) { @@ -186,7 +192,7 @@ protected override async Task RunInternalAsync(JobContext context) workingTasks.Remove(workItem); workItem.LastTaskInfo = taskStatus.Task; completedTasks.Add(workItem); - var targetCount = await client.CountAsync(d => d.Index(workItem.TargetIndex)); + var targetCount = await client.CountAsync(d => d.Indices(workItem.TargetIndex)); _logger.LogInformation("COMPLETED - {TargetIndex} ({TargetCount}) in {Duration:hh\\:mm} C:{Created} U:{Updated} D:{Deleted} X:{Conflicts} T:{Total} A:{Attempts} ID:{TaskId}", workItem.TargetIndex, targetCount.Count, duration, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total, workItem.Attempts, workItem.TaskId); } @@ -201,21 +207,27 @@ protected override async Task RunInternalAsync(JobContext context) _logger.LogInformation("----- REINDEX COMPLETE - I:{Completed}/{Total} T:{Duration:d\\.hh\\:mm} F:{Failed} R:{Retries}", completedTasks.Count, totalTasks, _timeProvider.GetUtcNow().UtcDateTime.Subtract(started), failedTasks.Count, retriesCount); foreach (var task in completedTasks) { - var status = task.LastTaskInfo.Status; - var duration = TimeSpan.FromMilliseconds(task.LastTaskInfo.RunningTimeInNanoseconds * 0.000001); + var status = task.LastTaskInfo.Status as ReindexStatus; + if (status is null) + continue; + + var duration = task.LastTaskInfo.RunningTimeInNanos; double progress = status.Total > 0 ? (status.Created + status.Updated + status.Deleted + status.VersionConflicts * 1.0) / status.Total : 0; - var targetCount = await client.CountAsync(d => d.Index(task.TargetIndex)); + var targetCount = await client.CountAsync(d => d.Indices(task.TargetIndex)); _logger.LogInformation("SUCCESS - {TargetIndex} ({TargetCount}) in {Duration:hh\\:mm} P:{Progress:F0}% C:{Created} U:{Updated} D:{Deleted} X:{Conflicts} T:{Total} A:{Attempts} ID:{TaskId}", task.TargetIndex, targetCount.Count, duration, progress, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total, task.Attempts, task.TaskId); } foreach (var task in failedTasks) { - var status = task.LastTaskInfo.Status; - var duration = TimeSpan.FromMilliseconds(task.LastTaskInfo.RunningTimeInNanoseconds * 0.000001); + var status = task.LastTaskInfo.Status as ReindexStatus; + if (status is null) + continue; + + var duration = task.LastTaskInfo.RunningTimeInNanos; double progress = status.Total > 0 ? (status.Created + status.Updated + status.Deleted + status.VersionConflicts * 1.0) / status.Total : 0; - var targetCount = await client.CountAsync(d => d.Index(task.TargetIndex)); + var targetCount = await client.CountAsync(d => d.Indices(task.TargetIndex)); _logger.LogCritical("FAILED - {TargetIndex} ({TargetCount}) in {Duration:hh\\:mm} P:{Progress:F0}% C:{Created} U:{Updated} D:{Deleted} X:{Conflicts} T:{Total} A:{Attempts} ID:{TaskId}", task.TargetIndex, targetCount.Count, duration, progress, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total, task.Attempts, task.TaskId); } @@ -227,7 +239,7 @@ protected override async Task RunInternalAsync(JobContext context) return JobResult.Success; } - private IRemoteSource ConfigureRemoteElasticSource(RemoteSourceDescriptor rsd) + private void ConfigureRemoteElasticSource(RemoteSourceDescriptor rsd) { var elasticOptions = _configuration.Options.ElasticsearchToMigrate; if (elasticOptions is null) @@ -236,7 +248,7 @@ private IRemoteSource ConfigureRemoteElasticSource(RemoteSourceDescriptor rsd) if (!String.IsNullOrEmpty(elasticOptions.UserName) && !String.IsNullOrEmpty(elasticOptions.Password)) rsd.Username(elasticOptions.UserName).Password(elasticOptions.Password); - return rsd.Host(new Uri(elasticOptions.ServerUrl)); + rsd.Host(elasticOptions.ServerUrl); } } diff --git a/src/Exceptionless.Core/Jobs/EventNotificationsJob.cs b/src/Exceptionless.Core/Jobs/EventNotificationsJob.cs index 0422a6b67e..8890398ce2 100644 --- a/src/Exceptionless.Core/Jobs/EventNotificationsJob.cs +++ b/src/Exceptionless.Core/Jobs/EventNotificationsJob.cs @@ -1,4 +1,3 @@ -using System.Text.Json; using Exceptionless.Core.Configuration; using Exceptionless.Core.Extensions; using Exceptionless.Core.Mail; @@ -13,6 +12,7 @@ using Foundatio.Queues; using Foundatio.Repositories; using Foundatio.Resilience; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Jobs; @@ -29,7 +29,7 @@ public class EventNotificationsJob : QueueJobBase private readonly IEventRepository _eventRepository; private readonly ICacheClient _cache; private readonly UserAgentParser _parser; - private readonly JsonSerializerOptions _jsonOptions; + private readonly ITextSerializer _serializer; public EventNotificationsJob(IQueue queue, SlackService slackService, @@ -41,7 +41,7 @@ public EventNotificationsJob(IQueue queue, IEventRepository eventRepository, ICacheClient cacheClient, UserAgentParser parser, - JsonSerializerOptions jsonOptions, + ITextSerializer serializer, TimeProvider timeProvider, IResiliencePolicyProvider resiliencePolicyProvider, ILoggerFactory loggerFactory) : base(queue, timeProvider, resiliencePolicyProvider, loggerFactory) @@ -55,7 +55,7 @@ public EventNotificationsJob(IQueue queue, _eventRepository = eventRepository; _cache = cacheClient; _parser = parser; - _jsonOptions = jsonOptions; + _serializer = serializer; } protected override async Task ProcessQueueEntryAsync(QueueEntryContext context) @@ -117,7 +117,7 @@ protected override async Task ProcessQueueEntryAsync(QueueEntryContex _logger.LogTrace("Settings: new error={ReportNewErrors} critical error={ReportCriticalErrors} regression={ReportEventRegressions} new={ReportNewEvents} critical={ReportCriticalEvents}", settings.ReportNewErrors, settings.ReportCriticalErrors, settings.ReportEventRegressions, settings.ReportNewEvents, settings.ReportCriticalEvents); _logger.LogTrace("Should process: new error={ShouldReportNewError} critical error={ShouldReportCriticalError} regression={ShouldReportRegression} new={ShouldReportNewEvent} critical={ShouldReportCriticalEvent}", shouldReportNewError, shouldReportCriticalError, shouldReportRegression, shouldReportNewEvent, shouldReportCriticalEvent); } - var request = ev.GetRequestInfo(_jsonOptions); + var request = ev.GetRequestInfo(_serializer, _logger); // check for known bots if the user has elected to not report them if (shouldReport && !String.IsNullOrEmpty(request?.UserAgent)) { diff --git a/src/Exceptionless.Core/Jobs/EventPostsJob.cs b/src/Exceptionless.Core/Jobs/EventPostsJob.cs index 0f771e9b64..0b7aaffec3 100644 --- a/src/Exceptionless.Core/Jobs/EventPostsJob.cs +++ b/src/Exceptionless.Core/Jobs/EventPostsJob.cs @@ -12,8 +12,8 @@ using Foundatio.Repositories; using Foundatio.Repositories.Exceptions; using Foundatio.Resilience; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; -using Newtonsoft.Json; namespace Exceptionless.Core.Jobs; @@ -29,10 +29,10 @@ public class EventPostsJob : QueueJobBase private readonly UsageService _usageService; private readonly IOrganizationRepository _organizationRepository; private readonly IProjectRepository _projectRepository; - private readonly JsonSerializerSettings _jsonSerializerSettings; + private readonly ITextSerializer _serializer; private readonly AppOptions _appOptions; - public EventPostsJob(IQueue queue, EventPostService eventPostService, EventParserPluginManager eventParserPluginManager, EventPipeline eventPipeline, UsageService usageService, IOrganizationRepository organizationRepository, IProjectRepository projectRepository, JsonSerializerSettings jsonSerializerSettings, AppOptions appOptions, TimeProvider timeProvider, + public EventPostsJob(IQueue queue, EventPostService eventPostService, EventParserPluginManager eventParserPluginManager, EventPipeline eventPipeline, UsageService usageService, IOrganizationRepository organizationRepository, IProjectRepository projectRepository, ITextSerializer serializer, AppOptions appOptions, TimeProvider timeProvider, IResiliencePolicyProvider resiliencePolicyProvider, ILoggerFactory loggerFactory) : base(queue, timeProvider, resiliencePolicyProvider, loggerFactory) { _eventPostService = eventPostService; @@ -41,7 +41,7 @@ public EventPostsJob(IQueue queue, EventPostService eventPostService, _usageService = usageService; _organizationRepository = organizationRepository; _projectRepository = projectRepository; - _jsonSerializerSettings = jsonSerializerSettings; + _serializer = serializer; _appOptions = appOptions; _maximumEventPostFileSize = _appOptions.MaximumEventPostSize + 1024; @@ -305,7 +305,7 @@ private async Task RetryEventsAsync(List eventsToRetry, EventPo { try { - var stream = new MemoryStream(ev.GetBytes(_jsonSerializerSettings)); + using var stream = new MemoryStream(ev.GetBytes(_serializer)); // Put this single event back into the queue so we can retry it separately. await _eventPostService.EnqueueAsync(new EventPost(false) diff --git a/src/Exceptionless.Core/Jobs/WebHooksJob.cs b/src/Exceptionless.Core/Jobs/WebHooksJob.cs index 40469d26f7..d8cd0a3536 100644 --- a/src/Exceptionless.Core/Jobs/WebHooksJob.cs +++ b/src/Exceptionless.Core/Jobs/WebHooksJob.cs @@ -1,6 +1,9 @@ using System.Net; +using System.Net.Http.Json; +using System.Text.Json; using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; +using Exceptionless.Core.Plugins.WebHook; using Exceptionless.Core.Queues.Models; using Exceptionless.Core.Repositories; using Exceptionless.Core.Services; @@ -10,8 +13,8 @@ using Foundatio.Queues; using Foundatio.Repositories; using Foundatio.Resilience; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; -using Newtonsoft.Json; namespace Exceptionless.Core.Jobs; @@ -32,7 +35,9 @@ public class WebHooksJob : QueueJobBase, IDisposable private readonly SlackService _slackService; private readonly IWebHookRepository _webHookRepository; private readonly ICacheClient _cacheClient; - private readonly JsonSerializerSettings _jsonSerializerSettings; + private readonly ITextSerializer _serializer; + private readonly JsonSerializerOptions _jsonOptions; + private readonly JsonSerializerOptions _versionOneJsonOptions; private readonly AppOptions _appOptions; private HttpClient? _client; @@ -42,14 +47,16 @@ private HttpClient Client get => _client ??= new HttpClient(); } - public WebHooksJob(IQueue queue, IProjectRepository projectRepository, SlackService slackService, IWebHookRepository webHookRepository, ICacheClient cacheClient, JsonSerializerSettings settings, AppOptions appOptions, TimeProvider timeProvider, + public WebHooksJob(IQueue queue, IProjectRepository projectRepository, SlackService slackService, IWebHookRepository webHookRepository, ICacheClient cacheClient, ITextSerializer serializer, JsonSerializerOptions jsonOptions, AppOptions appOptions, TimeProvider timeProvider, IResiliencePolicyProvider resiliencePolicyProvider, ILoggerFactory loggerFactory) : base(queue, timeProvider, resiliencePolicyProvider, loggerFactory) { _projectRepository = projectRepository; _slackService = slackService; _webHookRepository = webHookRepository; _cacheClient = cacheClient; - _jsonSerializerSettings = settings; + _serializer = serializer; + _jsonOptions = jsonOptions; + _versionOneJsonOptions = VersionOnePlugin.CreateJsonSerializerOptions(jsonOptions); _appOptions = appOptions; } @@ -85,17 +92,20 @@ protected override async Task ProcessQueueEntryAsync(QueueEntryContex HttpResponseMessage? response = null; try { - using (var timeoutCancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(5))) + using var timeoutCancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + using var postCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(context.CancellationToken, timeoutCancellationTokenSource.Token); + + var jsonOptions = body.Data switch { - using (var postCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(context.CancellationToken, timeoutCancellationTokenSource.Token)) - { - response = await Client.PostAsJsonAsync(body.Url, body.Data.ToJson(Formatting.Indented, _jsonSerializerSettings), postCancellationTokenSource.Token); - if (!response.IsSuccessStatusCode) - successful = false; - else if (consecutiveErrors > 0) - await cache.RemoveAllAsync(_cacheKeys); - } - } + VersionOnePlugin.VersionOneWebHookEvent or VersionOnePlugin.VersionOneWebHookStack => _versionOneJsonOptions, + _ => _jsonOptions + }; + + response = await Client.PostAsJsonAsync(body.Url, body.Data, jsonOptions, postCancellationTokenSource.Token); + if (!response.IsSuccessStatusCode) + successful = false; + else if (consecutiveErrors > 0) + await cache.RemoveAllAsync(_cacheKeys); } catch (OperationCanceledException ex) { @@ -172,7 +182,7 @@ private async Task IsEnabledAsync(WebHookNotification body) return webHook?.IsEnabled ?? false; case WebHookType.Slack: var project = await _projectRepository.GetByIdAsync(body.ProjectId, o => o.Cache()); - var token = project?.GetSlackToken(); + var token = project?.GetSlackToken(_serializer, _logger); return token is not null; } @@ -188,7 +198,7 @@ private async Task DisableIntegrationAsync(WebHookNotification body) break; case WebHookType.Slack: var project = await _projectRepository.GetByIdAsync(body.ProjectId); - var token = project?.GetSlackToken(); + var token = project?.GetSlackToken(_serializer, _logger); if (token is null) return; diff --git a/src/Exceptionless.Core/Mail/Mailer.cs b/src/Exceptionless.Core/Mail/Mailer.cs index b6498c1d45..da1c232a12 100644 --- a/src/Exceptionless.Core/Mail/Mailer.cs +++ b/src/Exceptionless.Core/Mail/Mailer.cs @@ -1,11 +1,11 @@ using System.Collections.Concurrent; -using System.Text.Json; using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Plugins.Formatting; using Exceptionless.Core.Queues.Models; using Exceptionless.DateTimeExtensions; using Foundatio.Queues; +using Foundatio.Serializer; using HandlebarsDotNet; using Microsoft.Extensions.Logging; @@ -18,16 +18,16 @@ public class Mailer : IMailer private readonly FormattingPluginManager _pluginManager; private readonly AppOptions _appOptions; private readonly TimeProvider _timeProvider; - private readonly JsonSerializerOptions _jsonOptions; + private readonly ITextSerializer _serializer; private readonly ILogger _logger; - public Mailer(IQueue queue, FormattingPluginManager pluginManager, JsonSerializerOptions jsonOptions, AppOptions appOptions, TimeProvider timeProvider, ILogger logger) + public Mailer(IQueue queue, FormattingPluginManager pluginManager, ITextSerializer serializer, AppOptions appOptions, TimeProvider timeProvider, ILogger logger) { _queue = queue; _pluginManager = pluginManager; _appOptions = appOptions; _timeProvider = timeProvider; - _jsonOptions = jsonOptions; + _serializer = serializer; _logger = logger; } @@ -59,7 +59,7 @@ public async Task SendEventNoticeAsync(User user, PersistentEvent ev, Proj }; AddDefaultFields(ev, result.Data); - AddUserInfo(ev, messageData, _jsonOptions); + AddUserInfo(ev, messageData); const string template = "event-notice"; await QueueMessageAsync(new MailMessage @@ -71,10 +71,10 @@ await QueueMessageAsync(new MailMessage return true; } - private static void AddUserInfo(PersistentEvent ev, Dictionary data, JsonSerializerOptions jsonOptions) + private void AddUserInfo(PersistentEvent ev, Dictionary data) { - var ud = ev.GetUserDescription(jsonOptions); - var ui = ev.GetUserIdentity(jsonOptions); + var ud = ev.GetUserDescription(_serializer, _logger); + var ui = ev.GetUserIdentity(_serializer, _logger); if (!String.IsNullOrEmpty(ud?.Description)) data["UserDescription"] = ud.Description; diff --git a/src/Exceptionless.Core/Migrations/001_UpdateIndexMappings.cs b/src/Exceptionless.Core/Migrations/001_UpdateIndexMappings.cs index c14657bb2d..92c518fbaf 100644 --- a/src/Exceptionless.Core/Migrations/001_UpdateIndexMappings.cs +++ b/src/Exceptionless.Core/Migrations/001_UpdateIndexMappings.cs @@ -1,15 +1,16 @@ +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Mapping; using Exceptionless.Core.Models; using Exceptionless.Core.Repositories.Configuration; using Foundatio.Repositories.Elasticsearch.Extensions; using Foundatio.Repositories.Migrations; using Microsoft.Extensions.Logging; -using Nest; namespace Exceptionless.Core.Migrations; public sealed class UpdateIndexMappings : MigrationBase { - private readonly IElasticClient _client; + private readonly ElasticsearchClient _client; private readonly ExceptionlessElasticConfiguration _config; public UpdateIndexMappings(ExceptionlessElasticConfiguration configuration, ILoggerFactory loggerFactory) : base(loggerFactory) @@ -26,56 +27,53 @@ public override async Task RunAsync(MigrationContext context) _logger.LogInformation("Start migration for adding index mappings..."); _logger.LogInformation("Updating Organization mappings..."); - var response = await _client.MapAsync(d => + var response = await _client.Indices.PutMappingAsync(d => { - d.Index(_config.Organizations.VersionedName); + d.Indices(_config.Organizations.VersionedName); d.Properties(p => p - .Date(f => f.Name(s => s.LastEventDateUtc)) - .Boolean(f => f.Name(s => s.IsDeleted)).FieldAlias(a => a.Path(p1 => p1.IsDeleted).Name("deleted"))); - - return d; + .Date(s => s.LastEventDateUtc) + .Boolean(s => s.IsDeleted) + .FieldAlias("deleted", new FieldAliasProperty { Path = "is_deleted" })); }); _logger.LogRequest(response); _logger.LogInformation("Setting Organization is_deleted=false..."); const string script = "ctx._source.is_deleted = false;"; await _config.Client.Indices.RefreshAsync(_config.Organizations.VersionedName); - var updateResponse = await _client.UpdateByQueryAsync(x => x.QueryOnQueryString("NOT _exists_:deleted").Script(s => s.Source(script).Lang(ScriptLang.Painless))); + var updateResponse = await _client.UpdateByQueryAsync(x => x.Query(q => q.QueryString(qs => qs.Query("NOT _exists_:deleted"))).Script(s => s.Source(script).Lang(ScriptLanguage.Painless))); _logger.LogRequest(updateResponse); _logger.LogInformation("Updating Project mappings..."); - response = await _client.MapAsync(d => + response = await _client.Indices.PutMappingAsync(d => { - d.Index(_config.Projects.VersionedName); + d.Indices(_config.Projects.VersionedName); d.Properties(p => p - .Date(f => f.Name(s => s.LastEventDateUtc)) - .Boolean(f => f.Name(s => s.IsDeleted)).FieldAlias(a => a.Path(p1 => p1.IsDeleted).Name("deleted"))); - - return d; + .Date(s => s.LastEventDateUtc) + .Boolean(s => s.IsDeleted) + .FieldAlias("deleted", new FieldAliasProperty { Path = "is_deleted" })); }); _logger.LogRequest(response); _logger.LogInformation("Setting Project is_deleted=false..."); await _config.Client.Indices.RefreshAsync(_config.Projects.VersionedName); - updateResponse = await _client.UpdateByQueryAsync(x => x.QueryOnQueryString("NOT _exists_:deleted").Script(s => s.Source(script).Lang(ScriptLang.Painless))); + updateResponse = await _client.UpdateByQueryAsync(x => x.Query(q => q.QueryString(qs => qs.Query("NOT _exists_:deleted"))).Script(s => s.Source(script).Lang(ScriptLanguage.Painless))); _logger.LogRequest(updateResponse); _logger.LogInformation("Updating Stack mappings..."); - response = await _client.MapAsync(d => + response = await _client.Indices.PutMappingAsync(d => { - d.Index(_config.Stacks.VersionedName); + d.Indices(_config.Stacks.VersionedName); d.Properties(p => p - .Keyword(f => f.Name(s => s.Status)) - .Date(f => f.Name(s => s.SnoozeUntilUtc)) - .Boolean(f => f.Name(s => s.IsDeleted)).FieldAlias(a => a.Path(p1 => p1.IsDeleted).Name("deleted"))); - - return d; + .Keyword(s => s.Status) + .Date(s => s.SnoozeUntilUtc) + .Boolean(s => s.IsDeleted) + .FieldAlias("deleted", new FieldAliasProperty { Path = "is_deleted" })); }); _logger.LogRequest(response); _logger.LogInformation("Setting Stack is_deleted=false..."); await _config.Client.Indices.RefreshAsync(_config.Stacks.VersionedName); - updateResponse = await _client.UpdateByQueryAsync(x => x.QueryOnQueryString("NOT _exists_:deleted").Script(s => s.Source(script).Lang(ScriptLang.Painless))); + updateResponse = await _client.UpdateByQueryAsync(x => x.Query(q => q.QueryString(qs => qs.Query("NOT _exists_:deleted"))).Script(s => s.Source(script).Lang(ScriptLanguage.Painless))); _logger.LogRequest(updateResponse); _logger.LogInformation("Finished adding mappings."); diff --git a/src/Exceptionless.Core/Migrations/002_SetStackStatus.cs b/src/Exceptionless.Core/Migrations/002_SetStackStatus.cs index cdf40112a6..c7a7ad41e1 100644 --- a/src/Exceptionless.Core/Migrations/002_SetStackStatus.cs +++ b/src/Exceptionless.Core/Migrations/002_SetStackStatus.cs @@ -1,17 +1,18 @@ using System.Diagnostics; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Core.ReindexRethrottle; using Exceptionless.Core.Models; using Exceptionless.Core.Repositories.Configuration; using Foundatio.Caching; using Foundatio.Repositories.Elasticsearch.Extensions; using Foundatio.Repositories.Migrations; using Microsoft.Extensions.Logging; -using Nest; namespace Exceptionless.Core.Migrations; public sealed class SetStackStatus : MigrationBase { - private readonly IElasticClient _client; + private readonly ElasticsearchClient _client; private readonly ExceptionlessElasticConfiguration _config; private readonly ICacheClient _cache; private readonly TimeProvider _timeProvider; @@ -37,9 +38,9 @@ public override async Task RunAsync(MigrationContext context) var sw = Stopwatch.StartNew(); const string script = "if (ctx._source.is_regressed == true) ctx._source.status = 'regressed'; else if (ctx._source.is_hidden == true) ctx._source.status = 'ignored'; else if (ctx._source.disable_notifications == true) ctx._source.status = 'ignored'; else if (ctx._source.is_fixed == true) ctx._source.status = 'fixed'; else if (ctx._source.containsKey('date_fixed')) ctx._source.status = 'fixed'; else ctx._source.status = 'open';"; var stackResponse = await _client.UpdateByQueryAsync(x => x - .QueryOnQueryString("NOT _exists_:status") - .Script(s => s.Source(script).Lang(ScriptLang.Painless)) - .Conflicts(Elasticsearch.Net.Conflicts.Proceed) + .Query(q => q.QueryString(qs => qs.Query("NOT _exists_:status"))) + .Script(s => s.Source(script).Lang(ScriptLanguage.Painless)) + .Conflicts(Conflicts.Proceed) .WaitForCompletion(false)); _logger.LogRequest(stackResponse, Microsoft.Extensions.Logging.LogLevel.Information); @@ -50,22 +51,22 @@ public override async Task RunAsync(MigrationContext context) do { attempts++; - var taskStatus = await _client.Tasks.GetTaskAsync(taskId); - var status = taskStatus.Task.Status; + var taskStatus = await _client.Tasks.GetAsync(taskId!.FullyQualifiedId); + var status = taskStatus.Task.Status as ReindexStatus; if (taskStatus.Completed) { // TODO: need to check to see if the task failed or completed successfully. Throw if it failed. - _logger.LogInformation("Script operation task ({TaskId}) completed: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total); - affectedRecords += status.Created + status.Updated + status.Deleted; + _logger.LogInformation("Script operation task ({TaskId}) completed: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status?.Created, status?.Updated, status?.Deleted, status?.VersionConflicts, status?.Total); + affectedRecords += (status?.Created ?? 0) + (status?.Updated ?? 0) + (status?.Deleted ?? 0); break; } - _logger.LogInformation("Checking script operation task ({TaskId}) status: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total); + _logger.LogInformation("Checking script operation task ({TaskId}) status: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status?.Created, status?.Updated, status?.Deleted, status?.VersionConflicts, status?.Total); var delay = TimeSpan.FromSeconds(attempts <= 5 ? 1 : 5); - await Task.Delay(delay, _timeProvider); + await Task.Delay(delay, _timeProvider, context.CancellationToken); } while (true); - _logger.LogInformation("Finished adding stack status: Time={Duration:d\\.hh\\:mm} Completed={Completed:N0} Total={Total:N0} Errors={Errors:N0}", sw.Elapsed, affectedRecords, stackResponse.Total, stackResponse.Failures.Count); + _logger.LogInformation("Finished adding stack status: Time={Duration:d\\.hh\\:mm} Completed={Completed:N0} Total={Total:N0} Errors={Errors:N0}", sw.Elapsed, affectedRecords, stackResponse.Total, stackResponse.Failures?.Count ?? 0); _logger.LogInformation("Invalidating Stack Cache"); await _cache.RemoveByPrefixAsync(nameof(Stack)); diff --git a/src/Exceptionless.Core/Migrations/FixDuplicateStacks.cs b/src/Exceptionless.Core/Migrations/FixDuplicateStacks.cs index 224b5416cc..101785ec16 100644 --- a/src/Exceptionless.Core/Migrations/FixDuplicateStacks.cs +++ b/src/Exceptionless.Core/Migrations/FixDuplicateStacks.cs @@ -1,3 +1,6 @@ +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Core.ReindexRethrottle; +using Elastic.Clients.Elasticsearch.QueryDsl; using Exceptionless.Core.Models; using Exceptionless.Core.Repositories; using Exceptionless.Core.Repositories.Configuration; @@ -8,14 +11,13 @@ using Foundatio.Repositories.Migrations; using Foundatio.Repositories.Models; using Microsoft.Extensions.Logging; -using Nest; using LogLevel = Microsoft.Extensions.Logging.LogLevel; namespace Exceptionless.Core.Migrations; public sealed class FixDuplicateStacks : MigrationBase { - private readonly IElasticClient _client; + private readonly ElasticsearchClient _client; private readonly ICacheClient _cache; private readonly IStackRepository _stackRepository; private readonly IEventRepository _eventRepository; @@ -39,12 +41,13 @@ public override async Task RunAsync(MigrationContext context) _logger.LogInformation("Getting duplicate stacks"); var duplicateStackAgg = await _client.SearchAsync(q => q - .QueryOnQueryString("is_deleted:false") + .Indices(_config.Stacks.VersionedName) + .Query(q => q.QueryString(qs => qs.Query("is_deleted:false"))) .Size(0) - .Aggregations(a => a.Terms("stacks", t => t.Field(f => f.DuplicateSignature).MinimumDocumentCount(2).Size(10000)))); + .AddAggregation("stacks", a => a.Terms(t => t.Field(f => f.DuplicateSignature).MinDocCount(2).Size(10000)))); _logger.LogRequest(duplicateStackAgg, LogLevel.Trace); - var buckets = duplicateStackAgg.Aggregations.Terms("stacks").Buckets; + var buckets = duplicateStackAgg.Aggregations?.GetStringTerms("stacks")?.Buckets ?? []; int total = buckets.Count; int processed = 0; int error = 0; @@ -62,7 +65,7 @@ public override async Task RunAsync(MigrationContext context) string? signature = null; try { - string[]? parts = duplicateSignature.Key.Split(':'); + string[]? parts = duplicateSignature.Key.ToString().Split(':'); if (parts.Length != 2) { _logger.LogError("Error parsing duplicate signature {DuplicateSignature}", duplicateSignature.Key); @@ -71,7 +74,7 @@ public override async Task RunAsync(MigrationContext context) projectId = parts[0]; signature = parts[1]; - var stacks = await _stackRepository.FindAsync(q => q.Project(projectId).FilterExpression($"signature_hash:{signature}")); + var stacks = await _stackRepository.FindAsync(q => q.Project(projectId).FieldEquals(s => s.SignatureHash, signature)); if (stacks.Documents.Count < 2) { _logger.LogError("Did not find multiple stacks with signature {SignatureHash} and project {ProjectId}", signature, projectId); @@ -117,11 +120,12 @@ public override async Task RunAsync(MigrationContext context) if (shouldUpdateEvents) { var response = await _client.UpdateByQueryAsync(u => u + .Indices(GetEventIndexPattern()) .Query(q => q.Bool(b => b.Must(m => m - .Terms(t => t.Field(f => f.StackId).Terms(duplicateStacks.Select(s => s.Id))) + .Terms(t => t.Field(f => f.StackId).Terms(new TermsQueryField(duplicateStacks.Select(s => (FieldValue)s.Id).ToList()))) ))) - .Script(s => s.Source($"ctx._source.stack_id = '{targetStack.Id}'").Lang(ScriptLang.Painless)) - .Conflicts(Elasticsearch.Net.Conflicts.Proceed) + .Script(s => s.Source($"ctx._source.stack_id = '{targetStack.Id}'").Lang(ScriptLanguage.Painless)) + .Conflicts(Conflicts.Proceed) .WaitForCompletion(false)); _logger.LogRequest(response, LogLevel.Trace); @@ -132,20 +136,20 @@ public override async Task RunAsync(MigrationContext context) do { attempts++; - var taskStatus = await _client.Tasks.GetTaskAsync(taskId); - var status = taskStatus.Task.Status; + var taskStatus = await _client.Tasks.GetAsync(taskId!.FullyQualifiedId); + var status = taskStatus.Task.Status as ReindexStatus; if (taskStatus.Completed) { // TODO: need to check to see if the task failed or completed successfully. Throw if it failed. if (_timeProvider.GetUtcNow().UtcDateTime.Subtract(taskStartedTime) > TimeSpan.FromSeconds(30)) - _logger.LogInformation("Script operation task ({TaskId}) completed: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total); + _logger.LogInformation("Script operation task ({TaskId}) completed: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status?.Created, status?.Updated, status?.Deleted, status?.VersionConflicts, status?.Total); - affectedRecords += status.Created + status.Updated + status.Deleted; + affectedRecords += (status?.Created ?? 0) + (status?.Updated ?? 0) + (status?.Deleted ?? 0); break; } if (_timeProvider.GetUtcNow().UtcDateTime.Subtract(taskStartedTime) > TimeSpan.FromSeconds(30)) - _logger.LogInformation("Checking script operation task ({TaskId}) status: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total); + _logger.LogInformation("Checking script operation task ({TaskId}) status: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status?.Created, status?.Updated, status?.Deleted, status?.VersionConflicts, status?.Total); var delay = TimeSpan.FromMilliseconds(50); if (attempts > 20) @@ -155,7 +159,7 @@ public override async Task RunAsync(MigrationContext context) else if (attempts > 5) delay = TimeSpan.FromMilliseconds(250); - await Task.Delay(delay, _timeProvider); + await Task.Delay(delay, context.CancellationToken); } while (true); _logger.LogInformation("Migrated stack events: Target={TargetId} Events={UpdatedEvents} Dupes={DuplicateIds}", targetStack.Id, affectedRecords, duplicateStacks.Select(s => s.Id)); @@ -179,12 +183,13 @@ public override async Task RunAsync(MigrationContext context) await _client.Indices.RefreshAsync(_config.Stacks.VersionedName); duplicateStackAgg = await _client.SearchAsync(q => q - .QueryOnQueryString("is_deleted:false") + .Indices(_config.Stacks.VersionedName) + .Query(q => q.QueryString(qs => qs.Query("is_deleted:false"))) .Size(0) - .Aggregations(a => a.Terms("stacks", t => t.Field(f => f.DuplicateSignature).MinimumDocumentCount(2).Size(10000)))); + .AddAggregation("stacks", a => a.Terms(t => t.Field(f => f.DuplicateSignature).MinDocCount(2).Size(10000)))); _logger.LogRequest(duplicateStackAgg, LogLevel.Trace); - buckets = duplicateStackAgg.Aggregations.Terms("stacks").Buckets; + buckets = duplicateStackAgg.Aggregations?.GetStringTerms("stacks")?.Buckets ?? []; total += buckets.Count; batch++; @@ -192,4 +197,9 @@ public override async Task RunAsync(MigrationContext context) await _cache.RemoveByPrefixAsync(nameof(Stack)); } } + + private string GetEventIndexPattern() + { + return $"{_config.Events.VersionedName}-*"; + } } diff --git a/src/Exceptionless.Core/Migrations/SetStackDuplicateSignature.cs b/src/Exceptionless.Core/Migrations/SetStackDuplicateSignature.cs index 435f3248ad..442bffaa8b 100644 --- a/src/Exceptionless.Core/Migrations/SetStackDuplicateSignature.cs +++ b/src/Exceptionless.Core/Migrations/SetStackDuplicateSignature.cs @@ -1,17 +1,18 @@ using System.Diagnostics; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Core.ReindexRethrottle; using Exceptionless.Core.Models; using Exceptionless.Core.Repositories.Configuration; using Foundatio.Caching; using Foundatio.Repositories.Elasticsearch.Extensions; using Foundatio.Repositories.Migrations; using Microsoft.Extensions.Logging; -using Nest; namespace Exceptionless.Core.Migrations; public sealed class SetStackDuplicateSignature : MigrationBase { - private readonly IElasticClient _client; + private readonly ElasticsearchClient _client; private readonly ExceptionlessElasticConfiguration _config; private readonly ICacheClient _cache; private readonly TimeProvider _timeProvider; @@ -33,12 +34,10 @@ public override async Task RunAsync(MigrationContext context) _logger.LogInformation("Done refreshing all indices"); _logger.LogInformation("Updating Stack mappings..."); - var response = await _client.MapAsync(d => + var response = await _client.Indices.PutMappingAsync(d => { - d.Index(_config.Stacks.VersionedName); - d.Properties(p => p.Keyword(f => f.Name(s => s.DuplicateSignature))); - - return d; + d.Indices(_config.Stacks.VersionedName); + d.Properties(p => p.Keyword(s => s.DuplicateSignature)); }); _logger.LogRequest(response); @@ -46,9 +45,9 @@ public override async Task RunAsync(MigrationContext context) var sw = Stopwatch.StartNew(); const string script = "ctx._source.duplicate_signature = ctx._source.project_id + ':' + ctx._source.signature_hash;"; var stackResponse = await _client.UpdateByQueryAsync(x => x - .QueryOnQueryString("NOT _exists_:duplicate_signature") - .Script(s => s.Source(script).Lang(ScriptLang.Painless)) - .Conflicts(Elasticsearch.Net.Conflicts.Proceed) + .Query(q => q.QueryString(qs => qs.Query("NOT _exists_:duplicate_signature"))) + .Script(s => s.Source(script).Lang(ScriptLanguage.Painless)) + .Conflicts(Conflicts.Proceed) .WaitForCompletion(false)); _logger.LogRequest(stackResponse, Microsoft.Extensions.Logging.LogLevel.Information); @@ -59,22 +58,22 @@ public override async Task RunAsync(MigrationContext context) do { attempts++; - var taskStatus = await _client.Tasks.GetTaskAsync(taskId); - var status = taskStatus.Task.Status; + var taskStatus = await _client.Tasks.GetAsync(taskId!.FullyQualifiedId); + var status = taskStatus.Task.Status as ReindexStatus; if (taskStatus.Completed) { // TODO: need to check to see if the task failed or completed successfully. Throw if it failed. - _logger.LogInformation("Script operation task ({TaskId}) completed: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total); - affectedRecords += status.Created + status.Updated + status.Deleted; + _logger.LogInformation("Script operation task ({TaskId}) completed: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status?.Created, status?.Updated, status?.Deleted, status?.VersionConflicts, status?.Total); + affectedRecords += (status?.Created ?? 0) + (status?.Updated ?? 0) + (status?.Deleted ?? 0); break; } - _logger.LogInformation("Checking script operation task ({TaskId}) status: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total); + _logger.LogInformation("Checking script operation task ({TaskId}) status: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status?.Created, status?.Updated, status?.Deleted, status?.VersionConflicts, status?.Total); var delay = TimeSpan.FromSeconds(attempts <= 5 ? 1 : 5); - await Task.Delay(delay, _timeProvider); + await Task.Delay(delay, _timeProvider, context.CancellationToken); } while (true); - _logger.LogInformation("Finished adding stack duplicate signature: Time={Duration:d\\.hh\\:mm} Completed={Completed:N0} Total={Total:N0} Errors={Errors:N0}", sw.Elapsed, affectedRecords, stackResponse.Total, stackResponse.Failures.Count); + _logger.LogInformation("Finished adding stack duplicate signature: Time={Duration:d\\.hh\\:mm} Completed={Completed:N0} Total={Total:N0} Errors={Errors:N0}", sw.Elapsed, affectedRecords, stackResponse.Total, stackResponse.Failures?.Count ?? 0); _logger.LogInformation("Invalidating Stack Cache"); await _cache.RemoveByPrefixAsync(nameof(Stack)); diff --git a/src/Exceptionless.Core/Migrations/UpdateEventUsage.cs b/src/Exceptionless.Core/Migrations/UpdateEventUsage.cs index 47c7f055f3..3860bd5848 100644 --- a/src/Exceptionless.Core/Migrations/UpdateEventUsage.cs +++ b/src/Exceptionless.Core/Migrations/UpdateEventUsage.cs @@ -1,4 +1,5 @@ using System.Diagnostics; +using Elastic.Clients.Elasticsearch; using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Repositories; @@ -8,7 +9,6 @@ using Foundatio.Repositories.Migrations; using Foundatio.Repositories.Models; using Microsoft.Extensions.Logging; -using Nest; namespace Exceptionless.Core.Migrations; @@ -116,7 +116,6 @@ private async Task UpdateProjectsUsageAsync(MigrationContext context, Organizati var projectResults = await _projectRepository.GetByOrganizationIdAsync(organization.Id, o => o.SoftDeleteMode(SoftDeleteQueryMode.All).SearchAfterPaging().PageLimit(100)); _logger.LogInformation("Updating usage for {ProjectTotal} projects(s)", projectResults.Total); - var sw = Stopwatch.StartNew(); while (projectResults.Documents.Count > 0 && !context.CancellationToken.IsCancellationRequested) { foreach (var project in projectResults.Documents) diff --git a/src/Exceptionless.Core/Models/Data/EnvironmentInfo.cs b/src/Exceptionless.Core/Models/Data/EnvironmentInfo.cs index 3d6bed4b81..bcab2bcc69 100644 --- a/src/Exceptionless.Core/Models/Data/EnvironmentInfo.cs +++ b/src/Exceptionless.Core/Models/Data/EnvironmentInfo.cs @@ -1,3 +1,4 @@ +using System.Text.Json.Serialization; using Exceptionless.Core.Extensions; namespace Exceptionless.Core.Models.Data; @@ -66,11 +67,13 @@ public class EnvironmentInfo : IData /// /// The OS name that the error occurred on. /// + [JsonPropertyName("o_s_name")] public string? OSName { get; set; } /// /// The OS version that the error occurred on. /// + [JsonPropertyName("o_s_version")] public string? OSVersion { get; set; } /// diff --git a/src/Exceptionless.Core/Models/Event.cs b/src/Exceptionless.Core/Models/Event.cs index 44101595b1..53db8ec17b 100644 --- a/src/Exceptionless.Core/Models/Event.cs +++ b/src/Exceptionless.Core/Models/Event.cs @@ -1,12 +1,15 @@ using System.ComponentModel.DataAnnotations; using System.Diagnostics; +using System.Text.Json; +using System.Text.Json.Serialization; using Exceptionless.Core.Extensions; +using Exceptionless.Core.Serialization; using MiniValidation; namespace Exceptionless.Core.Models; [DebuggerDisplay("Type: {Type}, Date: {Date}, Message: {Message}, Value: {Value}, Count: {Count}")] -public class Event : IData +public class Event : IData, IJsonOnDeserialized { /// /// The event type (ie. error, log message, feature usage). Check Event.KnownTypes for standard event types. @@ -59,11 +62,70 @@ public class Event : IData [SkipRecursion] public DataDictionary? Data { get; set; } = new(); + /// + /// Captures unknown JSON properties during deserialization. + /// These are merged into after deserialization. + /// Known data keys like "@error", "@request", "@environment" may appear at root level. + /// + [JsonExtensionData] + [JsonInclude] + internal Dictionary? ExtensionData { get; set; } + /// /// An optional identifier to be used for referencing this event instance at a later time. /// public string? ReferenceId { get; set; } + /// + /// Called after JSON deserialization to merge extension data into the Data dictionary. + /// This handles the case where known data keys like "@error", "@request", "@environment" + /// appear at the JSON root level instead of nested under "data". + /// + void IJsonOnDeserialized.OnDeserialized() + { + if (ExtensionData is { Count: > 0 }) + { + // STJ with SnakeCaseLower policy only matches case-insensitively against the + // policy-transformed name "reference_id". PascalCase "ReferenceId" and camelCase + // "referenceId" are entirely different strings (not just different casing) so they + // go to ExtensionData. Required for older .NET SDK versions (< 5.x) that submit + // events with PascalCase/camelCase property names. + if (ReferenceId is null) + { + if (ExtensionData.Remove("ReferenceId", out var refIdElement) || + ExtensionData.Remove("referenceId", out refIdElement)) + { + ReferenceId = refIdElement.GetString(); + } + } + + Data ??= []; + foreach (var kvp in ExtensionData) + { + object? value = JsonElementConverter.Convert(kvp.Value); + Data[GetDataKey(kvp.Key)] = value; + } + + ExtensionData = null; + } + } + + private string GetDataKey(string dataKey) + { + if (Data is null) + return dataKey; + + if (Data.ContainsKey(dataKey)) + dataKey = dataKey.StartsWith('@') ? "_" + dataKey : dataKey; + + int count = 1; + string key = dataKey; + while (Data.ContainsKey(key)) + key = dataKey + count++; + + return key; + } + protected bool Equals(Event other) { return String.Equals(Type, other.Type) && String.Equals(Source, other.Source) && Tags.CollectionEquals(other.Tags) && String.Equals(Message, other.Message) && String.Equals(Geo, other.Geo) && Value == other.Value && Equals(Data, other.Data); diff --git a/src/Exceptionless.Core/Models/Messaging/ReleaseNotification.cs b/src/Exceptionless.Core/Models/Messaging/ReleaseNotification.cs index b819fcd7d1..baab8744a0 100644 --- a/src/Exceptionless.Core/Models/Messaging/ReleaseNotification.cs +++ b/src/Exceptionless.Core/Models/Messaging/ReleaseNotification.cs @@ -1,4 +1,4 @@ -namespace Exceptionless.Core.Messaging.Models; +namespace Exceptionless.Core.Messaging.Models; public record ReleaseNotification { diff --git a/src/Exceptionless.Core/Models/SavedView.cs b/src/Exceptionless.Core/Models/SavedView.cs index 54793aec4d..22f264707f 100644 --- a/src/Exceptionless.Core/Models/SavedView.cs +++ b/src/Exceptionless.Core/Models/SavedView.cs @@ -1,4 +1,5 @@ using System.ComponentModel.DataAnnotations; +using System.Text.RegularExpressions; using Exceptionless.Core.Attributes; using Foundatio.Repositories.Models; @@ -8,9 +9,10 @@ namespace Exceptionless.Core.Models; /// A saved view captures filter, time range, and display settings for a dashboard page. /// Org-scoped; optionally user-private when UserId is set. /// -public record SavedView : IOwnedByOrganizationWithIdentity, IHaveDates +public partial record SavedView : IOwnedByOrganizationWithIdentity, IHaveDates { public const int MaxFilterDefinitionsLength = 100_000; + public const string SlugPattern = "^(?![a-f0-9]{24}$)[a-z0-9]+(?:-[a-z0-9]+)*$"; // Identity [ObjectId] @@ -63,7 +65,7 @@ public record SavedView : IOwnedByOrganizationWithIdentity, IHaveDates /// URL slug used to load this saved view. [Required] [MaxLength(100)] - [RegularExpression("^(?![a-f0-9]{24}$)[a-z0-9]+(?:-[a-z0-9]+)*$")] + [RegularExpression(SlugPattern)] public string Slug { get; set; } = null!; /// Date-math time range, e.g. "[now-7d TO now]". Null if no time constraint. @@ -85,4 +87,7 @@ public record SavedView : IOwnedByOrganizationWithIdentity, IHaveDates // Timestamps public DateTime CreatedUtc { get; set; } public DateTime UpdatedUtc { get; set; } + + [GeneratedRegex(SlugPattern)] + public static partial Regex SlugRegex(); } diff --git a/src/Exceptionless.Core/Models/SlackToken.cs b/src/Exceptionless.Core/Models/SlackToken.cs index 28d89e3a72..edd12d00da 100644 --- a/src/Exceptionless.Core/Models/SlackToken.cs +++ b/src/Exceptionless.Core/Models/SlackToken.cs @@ -1,5 +1,6 @@ -using System.Text.Json; -using Newtonsoft.Json; +using System.Text.Json.Serialization; +using Foundatio.Serializer; +using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Models; @@ -28,19 +29,19 @@ public SlackMessage(string text) Text = text; } - [JsonProperty("text")] + [JsonPropertyName("text")] public string Text { get; init; } - [JsonProperty("attachments")] + [JsonPropertyName("attachments")] public List Attachments { get; init; } = []; public class SlackAttachment { - public SlackAttachment(PersistentEvent ev, JsonSerializerOptions jsonOptions) + public SlackAttachment(PersistentEvent ev, ITextSerializer serializer, ILogger logger) { TimeStamp = ev.Date.ToUnixTimeSeconds(); - var ud = ev.GetUserDescription(jsonOptions); - var ui = ev.GetUserIdentity(jsonOptions); + var ud = ev.GetUserDescription(serializer, logger); + var ui = ev.GetUserIdentity(serializer, logger); Text = ud?.Description; string? displayName = null; @@ -67,34 +68,34 @@ public SlackAttachment(PersistentEvent ev, JsonSerializerOptions jsonOptions) } } - [JsonProperty("title")] + [JsonPropertyName("title")] public string? Title { get; init; } - [JsonProperty("text")] + [JsonPropertyName("text")] public string? Text { get; init; } - [JsonProperty("author_name")] + [JsonPropertyName("author_name")] public string? AuthorName { get; init; } - [JsonProperty("author_link")] + [JsonPropertyName("author_link")] public string? AuthorLink { get; init; } - [JsonProperty("author_icon")] + [JsonPropertyName("author_icon")] public string? AuthorIcon { get; init; } - [JsonProperty("color")] + [JsonPropertyName("color")] public string Color { get; set; } = "#5E9A00"; - [JsonProperty("fields")] + [JsonPropertyName("fields")] public List Fields { get; init; } = []; - [JsonProperty("mrkdwn_in")] + [JsonPropertyName("mrkdwn_in")] public string[] SupportedMarkdownFields { get; init; } = ["text", "fields"]; - [JsonProperty("ts")] + [JsonPropertyName("ts")] public long TimeStamp { get; init; } } public record SlackAttachmentFields { - [JsonProperty("title")] + [JsonPropertyName("title")] public string Title { get; init; } = null!; - [JsonProperty("value")] + [JsonPropertyName("value")] public string? Value { get; init; } - [JsonProperty("short")] + [JsonPropertyName("short")] public bool Short { get; init; } } } diff --git a/src/Exceptionless.Core/Models/Stack.cs b/src/Exceptionless.Core/Models/Stack.cs index a9a7589919..911835df31 100644 --- a/src/Exceptionless.Core/Models/Stack.cs +++ b/src/Exceptionless.Core/Models/Stack.cs @@ -5,7 +5,6 @@ using System.Text.Json.Serialization; using Exceptionless.Core.Attributes; using Foundatio.Repositories.Models; -using Newtonsoft.Json.Converters; namespace Exceptionless.Core.Models; @@ -155,7 +154,6 @@ public IEnumerable Validate(ValidationContext validationContex } [JsonConverter(typeof(JsonStringEnumConverter))] -[Newtonsoft.Json.JsonConverter(typeof(StringEnumConverter))] public enum StackStatus { [JsonStringEnumMemberName("open")] diff --git a/src/Exceptionless.Core/Models/SummaryData.cs b/src/Exceptionless.Core/Models/SummaryData.cs index 7433766a68..7419315017 100644 --- a/src/Exceptionless.Core/Models/SummaryData.cs +++ b/src/Exceptionless.Core/Models/SummaryData.cs @@ -4,5 +4,5 @@ public record SummaryData { public required string Id { get; set; } public required string TemplateKey { get; set; } - public required object Data { get; set; } + public object? Data { get; set; } } diff --git a/src/Exceptionless.Core/Plugins/EventParser/Default/JsonEventParserPlugin.cs b/src/Exceptionless.Core/Plugins/EventParser/Default/JsonEventParserPlugin.cs index b40c9520bb..cbf5f92c8e 100644 --- a/src/Exceptionless.Core/Plugins/EventParser/Default/JsonEventParserPlugin.cs +++ b/src/Exceptionless.Core/Plugins/EventParser/Default/JsonEventParserPlugin.cs @@ -1,19 +1,19 @@ -using Exceptionless.Core.Extensions; +using System.Text.Json; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Pipeline; using Microsoft.Extensions.Logging; -using Newtonsoft.Json; namespace Exceptionless.Core.Plugins.EventParser; [Priority(0)] public class JsonEventParserPlugin : PluginBase, IEventParserPlugin { - private readonly JsonSerializerSettings _settings; + private readonly JsonSerializerOptions _jsonOptions; - public JsonEventParserPlugin(AppOptions options, JsonSerializerSettings settings, ILoggerFactory loggerFactory) : base(options, loggerFactory) + public JsonEventParserPlugin(AppOptions options, JsonSerializerOptions jsonOptions, ILoggerFactory loggerFactory) : base(options, loggerFactory) { - _settings = settings; + _jsonOptions = jsonOptions; } public List? ParseEvents(string input, int apiVersion, string? userAgent) @@ -26,15 +26,30 @@ public JsonEventParserPlugin(AppOptions options, JsonSerializerSettings settings { case JsonType.Object: { - if (input.TryFromJson(out PersistentEvent? ev, _settings) && ev is not null) - events.Add(ev); + try + { + var ev = JsonSerializer.Deserialize(input, _jsonOptions); + if (ev is not null) + events.Add(ev); + } + catch (JsonException) + { + // Invalid JSON - ignore + } break; } case JsonType.Array: { - if (input.TryFromJson(out PersistentEvent[]? parsedEvents, _settings) && parsedEvents is { Length: > 0 }) - events.AddRange(parsedEvents); - + try + { + var parsedEvents = JsonSerializer.Deserialize(input, _jsonOptions); + if (parsedEvents is { Length: > 0 }) + events.AddRange(parsedEvents); + } + catch (JsonException) + { + // Invalid JSON - ignore + } break; } } diff --git a/src/Exceptionless.Core/Plugins/EventParser/Default/LegacyErrorParserPlugin.cs b/src/Exceptionless.Core/Plugins/EventParser/Default/LegacyErrorParserPlugin.cs index 5d8c337249..8500c52329 100644 --- a/src/Exceptionless.Core/Plugins/EventParser/Default/LegacyErrorParserPlugin.cs +++ b/src/Exceptionless.Core/Plugins/EventParser/Default/LegacyErrorParserPlugin.cs @@ -1,9 +1,9 @@ -using Exceptionless.Core.Extensions; +using System.Text.Json; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Pipeline; using Exceptionless.Core.Plugins.EventUpgrader; using Microsoft.Extensions.Logging; -using Newtonsoft.Json; namespace Exceptionless.Core.Plugins.EventParser; @@ -11,12 +11,12 @@ namespace Exceptionless.Core.Plugins.EventParser; public class LegacyErrorParserPlugin : PluginBase, IEventParserPlugin { private readonly EventUpgraderPluginManager _manager; - private readonly JsonSerializerSettings _settings; + private readonly JsonSerializerOptions _jsonOptions; - public LegacyErrorParserPlugin(EventUpgraderPluginManager manager, JsonSerializerSettings settings, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) + public LegacyErrorParserPlugin(EventUpgraderPluginManager manager, JsonSerializerOptions jsonOptions, AppOptions appOptions, ILoggerFactory loggerFactory) : base(appOptions, loggerFactory) { _manager = manager; - _settings = settings; + _jsonOptions = jsonOptions; } public List? ParseEvents(string input, int apiVersion, string? userAgent) @@ -29,7 +29,7 @@ public LegacyErrorParserPlugin(EventUpgraderPluginManager manager, JsonSerialize var ctx = new EventUpgraderContext(input); _manager.Upgrade(ctx); - return ctx.Documents.FromJson(_settings); + return ctx.Documents.ToList(_jsonOptions); } catch (Exception ex) { diff --git a/src/Exceptionless.Core/Plugins/EventProcessor/Default/03_ManualStackingPlugin.cs b/src/Exceptionless.Core/Plugins/EventProcessor/Default/03_ManualStackingPlugin.cs index 4b17fd8dc3..8a536a7c1e 100644 --- a/src/Exceptionless.Core/Plugins/EventProcessor/Default/03_ManualStackingPlugin.cs +++ b/src/Exceptionless.Core/Plugins/EventProcessor/Default/03_ManualStackingPlugin.cs @@ -1,6 +1,6 @@ -using System.Text.Json; -using Exceptionless.Core.Extensions; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Pipeline; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.EventProcessor; @@ -8,16 +8,16 @@ namespace Exceptionless.Core.Plugins.EventProcessor; [Priority(3)] public sealed class ManualStackingPlugin : EventProcessorPluginBase { - private readonly JsonSerializerOptions _jsonOptions; + private readonly ITextSerializer _serializer; - public ManualStackingPlugin(JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) + public ManualStackingPlugin(ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) { - _jsonOptions = jsonOptions; + _serializer = serializer; } public override Task EventProcessingAsync(EventContext context) { - var msi = context.Event.GetManualStackingInfo(_jsonOptions); + var msi = context.Event.GetManualStackingInfo(_serializer, _logger); if (msi?.SignatureData is not null) { foreach (var kvp in msi.SignatureData) diff --git a/src/Exceptionless.Core/Plugins/EventProcessor/Default/0_ThrottleBotsPlugin.cs b/src/Exceptionless.Core/Plugins/EventProcessor/Default/0_ThrottleBotsPlugin.cs index 36610ef3a8..3b18c78bcd 100644 --- a/src/Exceptionless.Core/Plugins/EventProcessor/Default/0_ThrottleBotsPlugin.cs +++ b/src/Exceptionless.Core/Plugins/EventProcessor/Default/0_ThrottleBotsPlugin.cs @@ -1,11 +1,11 @@ -using System.Text.Json; -using Exceptionless.Core.Extensions; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models.WorkItems; using Exceptionless.Core.Pipeline; using Exceptionless.DateTimeExtensions; using Foundatio.Caching; using Foundatio.Jobs; using Foundatio.Queues; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.EventProcessor; @@ -16,15 +16,15 @@ public sealed class ThrottleBotsPlugin : EventProcessorPluginBase private readonly ICacheClient _cache; private readonly IQueue _workItemQueue; private readonly TimeProvider _timeProvider; - private readonly JsonSerializerOptions _jsonOptions; + private readonly ITextSerializer _serializer; private readonly TimeSpan _throttlingPeriod = TimeSpan.FromMinutes(5); public ThrottleBotsPlugin(ICacheClient cacheClient, IQueue workItemQueue, - JsonSerializerOptions jsonOptions, TimeProvider timeProvider, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) + ITextSerializer serializer, TimeProvider timeProvider, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) { _cache = cacheClient; _workItemQueue = workItemQueue; - _jsonOptions = jsonOptions; + _serializer = serializer; _timeProvider = timeProvider; } @@ -38,7 +38,7 @@ public override async Task EventBatchProcessingAsync(ICollection c return; // Throttle errors by client ip address to no more than X every 5 minutes. - var clientIpAddressGroups = contexts.GroupBy(c => c.Event.GetRequestInfo(_jsonOptions)?.ClientIpAddress); + var clientIpAddressGroups = contexts.GroupBy(c => c.Event.GetRequestInfo(_serializer, _logger)?.ClientIpAddress); foreach (var clientIpAddressGroup in clientIpAddressGroups) { if (String.IsNullOrEmpty(clientIpAddressGroup.Key) || clientIpAddressGroup.Key.IsPrivateNetwork()) diff --git a/src/Exceptionless.Core/Plugins/EventProcessor/Default/10_NotFoundPlugin.cs b/src/Exceptionless.Core/Plugins/EventProcessor/Default/10_NotFoundPlugin.cs index 99599dcaa1..189f1e895a 100644 --- a/src/Exceptionless.Core/Plugins/EventProcessor/Default/10_NotFoundPlugin.cs +++ b/src/Exceptionless.Core/Plugins/EventProcessor/Default/10_NotFoundPlugin.cs @@ -1,7 +1,7 @@ -using System.Text.Json; -using Exceptionless.Core.Extensions; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Pipeline; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.EventProcessor; @@ -9,11 +9,11 @@ namespace Exceptionless.Core.Plugins.EventProcessor; [Priority(10)] public sealed class NotFoundPlugin : EventProcessorPluginBase { - private readonly JsonSerializerOptions _jsonOptions; + private readonly ITextSerializer _serializer; - public NotFoundPlugin(JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) + public NotFoundPlugin(ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) { - _jsonOptions = jsonOptions; + _serializer = serializer; } public override Task EventProcessingAsync(EventContext context) @@ -24,7 +24,7 @@ public override Task EventProcessingAsync(EventContext context) context.Event.Data.Remove(Event.KnownDataKeys.EnvironmentInfo); context.Event.Data.Remove(Event.KnownDataKeys.TraceLog); - var req = context.Event.GetRequestInfo(_jsonOptions); + var req = context.Event.GetRequestInfo(_serializer, _logger); if (req is null) return Task.CompletedTask; diff --git a/src/Exceptionless.Core/Plugins/EventProcessor/Default/20_ErrorPlugin.cs b/src/Exceptionless.Core/Plugins/EventProcessor/Default/20_ErrorPlugin.cs index f94a57e519..60773ed2c1 100644 --- a/src/Exceptionless.Core/Plugins/EventProcessor/Default/20_ErrorPlugin.cs +++ b/src/Exceptionless.Core/Plugins/EventProcessor/Default/20_ErrorPlugin.cs @@ -1,8 +1,8 @@ -using System.Text.Json; -using Exceptionless.Core.Extensions; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Pipeline; using Exceptionless.Core.Utility; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.EventProcessor; @@ -10,11 +10,11 @@ namespace Exceptionless.Core.Plugins.EventProcessor; [Priority(20)] public sealed class ErrorPlugin : EventProcessorPluginBase { - private readonly JsonSerializerOptions _jsonOptions; + private readonly ITextSerializer _serializer; - public ErrorPlugin(JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) + public ErrorPlugin(ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) { - _jsonOptions = jsonOptions; + _serializer = serializer; } public override Task EventProcessingAsync(EventContext context) @@ -22,7 +22,7 @@ public override Task EventProcessingAsync(EventContext context) if (!context.Event.IsError()) return Task.CompletedTask; - var error = context.Event.GetError(_jsonOptions); + var error = context.Event.GetError(_serializer, _logger); if (error is null) return Task.CompletedTask; @@ -40,7 +40,7 @@ public override Task EventProcessingAsync(EventContext context) if (context.HasProperty("UserNamespaces")) userNamespaces = context.GetProperty("UserNamespaces")?.SplitAndTrim([',']); - var signature = new ErrorSignature(error, _jsonOptions, userNamespaces, userCommonMethods); + var signature = new ErrorSignature(error, _serializer, userNamespaces, userCommonMethods); if (signature.SignatureInfo.Count <= 0) return Task.CompletedTask; @@ -50,6 +50,12 @@ public override Task EventProcessingAsync(EventContext context) targetInfo.AddItemIfNotEmpty("Message", stackingTarget.Error.Message); error.SetTargetInfo(targetInfo); + + // Write the mutated error back to Event.Data so pipeline changes (e.g., @target) + // persist. GetValue() deserializes a disconnected copy; without this write-back, + // SetTargetInfo mutations are lost when the event is later serialized for storage. + context.Event.SetError(error); + foreach (string key in signature.SignatureInfo.Keys) context.StackSignatureData.Add(key, signature.SignatureInfo[key]); diff --git a/src/Exceptionless.Core/Plugins/EventProcessor/Default/30_SimpleErrorPlugin.cs b/src/Exceptionless.Core/Plugins/EventProcessor/Default/30_SimpleErrorPlugin.cs index 5d3da08178..9bf334e303 100644 --- a/src/Exceptionless.Core/Plugins/EventProcessor/Default/30_SimpleErrorPlugin.cs +++ b/src/Exceptionless.Core/Plugins/EventProcessor/Default/30_SimpleErrorPlugin.cs @@ -1,7 +1,7 @@ -using System.Text.Json; -using Exceptionless.Core.Extensions; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Pipeline; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.EventProcessor; @@ -9,11 +9,11 @@ namespace Exceptionless.Core.Plugins.EventProcessor; [Priority(30)] public sealed class SimpleErrorPlugin : EventProcessorPluginBase { - private readonly JsonSerializerOptions _jsonOptions; + private readonly ITextSerializer _serializer; - public SimpleErrorPlugin(JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) + public SimpleErrorPlugin(ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) { - _jsonOptions = jsonOptions; + _serializer = serializer; } public override Task EventProcessingAsync(EventContext context) @@ -21,7 +21,7 @@ public override Task EventProcessingAsync(EventContext context) if (!context.Event.IsError()) return Task.CompletedTask; - var error = context.Event.GetSimpleError(_jsonOptions); + var error = context.Event.GetSimpleError(_serializer, _logger); if (error is null) return Task.CompletedTask; @@ -39,6 +39,10 @@ public override Task EventProcessingAsync(EventContext context) context.StackSignatureData.Add("StackTrace", error.StackTrace.ToSHA1()); error.SetTargetInfo(new SettingsDictionary(context.StackSignatureData)); + + // Write the mutated error back so @target persists (see ErrorPlugin for rationale). + context.Event.SetSimpleError(error); + return Task.CompletedTask; } } diff --git a/src/Exceptionless.Core/Plugins/EventProcessor/Default/40_RequestInfoPlugin.cs b/src/Exceptionless.Core/Plugins/EventProcessor/Default/40_RequestInfoPlugin.cs index 56d66b938f..7b9ecc2b14 100644 --- a/src/Exceptionless.Core/Plugins/EventProcessor/Default/40_RequestInfoPlugin.cs +++ b/src/Exceptionless.Core/Plugins/EventProcessor/Default/40_RequestInfoPlugin.cs @@ -1,9 +1,9 @@ -using System.Text.Json; -using Exceptionless.Core.Extensions; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Models.Data; using Exceptionless.Core.Pipeline; using Exceptionless.Core.Utility; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.EventProcessor; @@ -25,12 +25,12 @@ public sealed class RequestInfoPlugin : EventProcessorPluginBase ]; private readonly UserAgentParser _parser; - private readonly JsonSerializerOptions _jsonOptions; + private readonly ITextSerializer _serializer; - public RequestInfoPlugin(UserAgentParser parser, JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) + public RequestInfoPlugin(UserAgentParser parser, ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) { _parser = parser; - _jsonOptions = jsonOptions; + _serializer = serializer; } public override async Task EventBatchProcessingAsync(ICollection contexts) @@ -39,13 +39,13 @@ public override async Task EventBatchProcessingAsync(ICollection c var exclusions = DefaultExclusions.Union(project.Configuration.Settings.GetStringCollection(SettingsDictionary.KnownKeys.DataExclusions)).ToList(); foreach (var context in contexts) { - var request = context.Event.GetRequestInfo(_jsonOptions); + var request = context.Event.GetRequestInfo(_serializer, _logger); if (request is null) continue; if (context.IncludePrivateInformation) { - var submissionClient = context.Event.GetSubmissionClient(_jsonOptions); + var submissionClient = context.Event.GetSubmissionClient(_serializer, _logger); AddClientIpAddress(request, submissionClient); } else @@ -57,7 +57,7 @@ public override async Task EventBatchProcessingAsync(ICollection c } await SetBrowserOsAndDeviceFromUserAgent(request, context); - context.Event.AddRequestInfo(request.ApplyDataExclusions(exclusions, MAX_VALUE_LENGTH)); + context.Event.AddRequestInfo(request.ApplyDataExclusions(_serializer, exclusions, MAX_VALUE_LENGTH)); } } diff --git a/src/Exceptionless.Core/Plugins/EventProcessor/Default/45_EnvironmentInfoPlugin.cs b/src/Exceptionless.Core/Plugins/EventProcessor/Default/45_EnvironmentInfoPlugin.cs index e3fdcd23b6..a3025167ae 100644 --- a/src/Exceptionless.Core/Plugins/EventProcessor/Default/45_EnvironmentInfoPlugin.cs +++ b/src/Exceptionless.Core/Plugins/EventProcessor/Default/45_EnvironmentInfoPlugin.cs @@ -1,7 +1,7 @@ -using System.Text.Json; -using Exceptionless.Core.Extensions; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models.Data; using Exceptionless.Core.Pipeline; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.EventProcessor.Default; @@ -9,22 +9,22 @@ namespace Exceptionless.Core.Plugins.EventProcessor.Default; [Priority(45)] public sealed class EnvironmentInfoPlugin : EventProcessorPluginBase { - private readonly JsonSerializerOptions _jsonOptions; + private readonly ITextSerializer _serializer; - public EnvironmentInfoPlugin(JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) + public EnvironmentInfoPlugin(ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) { - _jsonOptions = jsonOptions; + _serializer = serializer; } public override Task EventProcessingAsync(EventContext context) { - var environment = context.Event.GetEnvironmentInfo(_jsonOptions); + var environment = context.Event.GetEnvironmentInfo(_serializer, _logger); if (environment is null) return Task.CompletedTask; if (context.IncludePrivateInformation) { - var submissionClient = context.Event.GetSubmissionClient(_jsonOptions); + var submissionClient = context.Event.GetSubmissionClient(_serializer, _logger); AddClientIpAddress(environment, submissionClient); } else diff --git a/src/Exceptionless.Core/Plugins/EventProcessor/Default/50_GeoPlugin.cs b/src/Exceptionless.Core/Plugins/EventProcessor/Default/50_GeoPlugin.cs index 13bea073bf..cd87f6f2b0 100644 --- a/src/Exceptionless.Core/Plugins/EventProcessor/Default/50_GeoPlugin.cs +++ b/src/Exceptionless.Core/Plugins/EventProcessor/Default/50_GeoPlugin.cs @@ -1,8 +1,8 @@ -using System.Text.Json; -using Exceptionless.Core.Extensions; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Geo; using Exceptionless.Core.Models; using Exceptionless.Core.Pipeline; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.EventProcessor.Default; @@ -11,12 +11,12 @@ namespace Exceptionless.Core.Plugins.EventProcessor.Default; public sealed class GeoPlugin : EventProcessorPluginBase { private readonly IGeoIpService _geoIpService; - private readonly JsonSerializerOptions _jsonOptions; + private readonly ITextSerializer _serializer; - public GeoPlugin(IGeoIpService geoIpService, JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) + public GeoPlugin(IGeoIpService geoIpService, ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) { _geoIpService = geoIpService; - _jsonOptions = jsonOptions; + _serializer = serializer; } public override Task EventBatchProcessingAsync(ICollection contexts) @@ -35,7 +35,7 @@ public override Task EventBatchProcessingAsync(ICollection context // The geo coordinates are all the same, set the location from the result of any of the ip addresses. if (!String.IsNullOrEmpty(group.Key)) { - var ips = group.SelectMany(c => c.Event.GetIpAddresses(_jsonOptions)).Union(new[] { group.First().EventPostInfo?.IpAddress }).Distinct().ToList(); + var ips = group.SelectMany(c => c.Event.GetIpAddresses(_serializer, _logger)).Union(new[] { group.First().EventPostInfo?.IpAddress }).Distinct().ToList(); if (ips.Count > 0) tasks.Add(UpdateGeoInformationAsync(group, ips)); continue; @@ -44,7 +44,7 @@ public override Task EventBatchProcessingAsync(ICollection context // Each event in the group could be a different user; foreach (var context in group) { - var ips = context.Event.GetIpAddresses(_jsonOptions).Union(new[] { context.EventPostInfo?.IpAddress }).ToList(); + var ips = context.Event.GetIpAddresses(_serializer, _logger).Union(new[] { context.EventPostInfo?.IpAddress }).ToList(); if (ips.Count > 0) tasks.Add(UpdateGeoInformationAsync(context, ips)); } diff --git a/src/Exceptionless.Core/Plugins/EventProcessor/Default/70_SessionPlugin.cs b/src/Exceptionless.Core/Plugins/EventProcessor/Default/70_SessionPlugin.cs index 42b0169be7..580831e116 100644 --- a/src/Exceptionless.Core/Plugins/EventProcessor/Default/70_SessionPlugin.cs +++ b/src/Exceptionless.Core/Plugins/EventProcessor/Default/70_SessionPlugin.cs @@ -1,10 +1,10 @@ -using System.Text.Json; -using Exceptionless.Core.Extensions; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Pipeline; using Exceptionless.Core.Repositories; using Foundatio.Caching; using Foundatio.Repositories.Utility; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.EventProcessor.Default; @@ -18,21 +18,21 @@ public sealed class SessionPlugin : EventProcessorPluginBase private readonly UpdateStatsAction _updateStats; private readonly AssignToStackAction _assignToStack; private readonly LocationPlugin _locationPlugin; - private readonly JsonSerializerOptions _jsonOptions; + private readonly ITextSerializer _serializer; - public SessionPlugin(ICacheClient cacheClient, IEventRepository eventRepository, AssignToStackAction assignToStack, UpdateStatsAction updateStats, LocationPlugin locationPlugin, JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) + public SessionPlugin(ICacheClient cacheClient, IEventRepository eventRepository, AssignToStackAction assignToStack, UpdateStatsAction updateStats, LocationPlugin locationPlugin, ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) { _cache = new ScopedCacheClient(cacheClient, "session"); _eventRepository = eventRepository; _assignToStack = assignToStack; _updateStats = updateStats; _locationPlugin = locationPlugin; - _jsonOptions = jsonOptions; + _serializer = serializer; } public override Task EventBatchProcessingAsync(ICollection contexts) { - var autoSessionEvents = contexts.Where(c => !String.IsNullOrWhiteSpace(c.Event.GetUserIdentity(_jsonOptions)?.Identity) && String.IsNullOrEmpty(c.Event.GetSessionId())).ToList(); + var autoSessionEvents = contexts.Where(c => !String.IsNullOrWhiteSpace(c.Event.GetUserIdentity(_serializer, _logger)?.Identity) && String.IsNullOrEmpty(c.Event.GetSessionId())).ToList(); var manualSessionsEvents = contexts.Where(c => !String.IsNullOrEmpty(c.Event.GetSessionId())).ToList(); return Task.WhenAll( @@ -125,7 +125,7 @@ private async Task ProcessAutoSessionsAsync(ICollection contexts) { var identityGroups = contexts .OrderBy(c => c.Event.Date) - .GroupBy(c => c.Event.GetUserIdentity(_jsonOptions)?.Identity); + .GroupBy(c => c.Event.GetUserIdentity(_serializer, _logger)?.Identity); foreach (var identityGroup in identityGroups) { @@ -286,7 +286,7 @@ private Task SetIdentitySessionIdAsync(string projectId, string identity, private async Task CreateSessionStartEventAsync(EventContext startContext, DateTime? lastActivityUtc, bool? isSessionEnd) { - var startEvent = startContext.Event.ToSessionStartEvent(_jsonOptions, lastActivityUtc, isSessionEnd, startContext.Organization.HasPremiumFeatures, startContext.IncludePrivateInformation); + var startEvent = startContext.Event.ToSessionStartEvent(_serializer, _logger, lastActivityUtc, isSessionEnd, startContext.Organization.HasPremiumFeatures, startContext.IncludePrivateInformation); var startEventContexts = new List { new(startEvent, startContext.Organization, startContext.Project) }; diff --git a/src/Exceptionless.Core/Plugins/EventProcessor/Default/80_AngularPlugin.cs b/src/Exceptionless.Core/Plugins/EventProcessor/Default/80_AngularPlugin.cs index 5d89be33e9..031b835622 100644 --- a/src/Exceptionless.Core/Plugins/EventProcessor/Default/80_AngularPlugin.cs +++ b/src/Exceptionless.Core/Plugins/EventProcessor/Default/80_AngularPlugin.cs @@ -1,7 +1,7 @@ -using System.Text.Json; -using Exceptionless.Core.Extensions; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Pipeline; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.EventProcessor; @@ -9,11 +9,11 @@ namespace Exceptionless.Core.Plugins.EventProcessor; [Priority(80)] public sealed class AngularPlugin : EventProcessorPluginBase { - private readonly JsonSerializerOptions _jsonOptions; + private readonly ITextSerializer _serializer; - public AngularPlugin(JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) + public AngularPlugin(ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) { - _jsonOptions = jsonOptions; + _serializer = serializer; } public override Task EventProcessingAsync(EventContext context) @@ -21,7 +21,7 @@ public override Task EventProcessingAsync(EventContext context) if (!context.Event.IsError()) return Task.CompletedTask; - var error = context.Event.GetError(_jsonOptions); + var error = context.Event.GetError(_serializer, _logger); if (error is null) return Task.CompletedTask; @@ -43,6 +43,9 @@ public override Task EventProcessingAsync(EventContext context) context.StackSignatureData.Add("Source", "unhandledRejection"); error.SetTargetInfo(new SettingsDictionary(context.StackSignatureData)); + + // Write the mutated error back so @target persists (see ErrorPlugin for rationale). + context.Event.SetError(error); } return Task.CompletedTask; diff --git a/src/Exceptionless.Core/Plugins/EventProcessor/Default/90_RemovePrivateInformationPlugin.cs b/src/Exceptionless.Core/Plugins/EventProcessor/Default/90_RemovePrivateInformationPlugin.cs index 757ee3cfdd..6bb89b3823 100644 --- a/src/Exceptionless.Core/Plugins/EventProcessor/Default/90_RemovePrivateInformationPlugin.cs +++ b/src/Exceptionless.Core/Plugins/EventProcessor/Default/90_RemovePrivateInformationPlugin.cs @@ -1,5 +1,5 @@ -using System.Text.Json; -using Exceptionless.Core.Pipeline; +using Exceptionless.Core.Pipeline; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.EventProcessor.Default; @@ -7,11 +7,11 @@ namespace Exceptionless.Core.Plugins.EventProcessor.Default; [Priority(90)] public sealed class RemovePrivateInformationPlugin : EventProcessorPluginBase { - private readonly JsonSerializerOptions _jsonOptions; + private readonly ITextSerializer _serializer; - public RemovePrivateInformationPlugin(JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) + public RemovePrivateInformationPlugin(ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) { - _jsonOptions = jsonOptions; + _serializer = serializer; } public override Task EventProcessingAsync(EventContext context) @@ -21,7 +21,7 @@ public override Task EventProcessingAsync(EventContext context) context.Event.RemoveUserIdentity(); - var description = context.Event.GetUserDescription(_jsonOptions); + var description = context.Event.GetUserDescription(_serializer, _logger); if (description is not null) { description.EmailAddress = null; diff --git a/src/Exceptionless.Core/Plugins/EventUpgrader/Default/GetVersion.cs b/src/Exceptionless.Core/Plugins/EventUpgrader/Default/GetVersion.cs index 229a47452f..41844c6175 100644 --- a/src/Exceptionless.Core/Plugins/EventUpgrader/Default/GetVersion.cs +++ b/src/Exceptionless.Core/Plugins/EventUpgrader/Default/GetVersion.cs @@ -1,7 +1,7 @@ -using Exceptionless.Core.Extensions; +using System.Text.Json.Nodes; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Pipeline; using Microsoft.Extensions.Logging; -using Newtonsoft.Json.Linq; namespace Exceptionless.Core.Plugins.EventUpgrader; @@ -15,14 +15,14 @@ public void Upgrade(EventUpgraderContext ctx) if (ctx.Version is not null) return; - if (ctx.Documents.Count == 0 || !ctx.Documents.First().HasValues) + if (ctx.Documents.Count == 0 || !ctx.Documents.First().HasValues()) { ctx.Version = new Version(); return; } var doc = ctx.Documents.First(); - if (!(doc["ExceptionlessClientInfo"] is JObject { HasValues: true } clientInfo) || clientInfo["Version"] is null) + if (doc is not JsonObject docObj || docObj["ExceptionlessClientInfo"] is not JsonObject { Count: > 0 } clientInfo || clientInfo["Version"] is null) { ctx.Version = new Version(); return; diff --git a/src/Exceptionless.Core/Plugins/EventUpgrader/Default/V1R500_EventUpgrade.cs b/src/Exceptionless.Core/Plugins/EventUpgrader/Default/V1R500_EventUpgrade.cs index 3b69f5bf7f..c076431532 100644 --- a/src/Exceptionless.Core/Plugins/EventUpgrader/Default/V1R500_EventUpgrade.cs +++ b/src/Exceptionless.Core/Plugins/EventUpgrader/Default/V1R500_EventUpgrade.cs @@ -1,6 +1,7 @@ -using Exceptionless.Core.Pipeline; +using System.Globalization; +using System.Text.Json.Nodes; +using Exceptionless.Core.Pipeline; using Microsoft.Extensions.Logging; -using Newtonsoft.Json.Linq; namespace Exceptionless.Core.Plugins.EventUpgrader; @@ -19,14 +20,14 @@ public void Upgrade(EventUpgraderContext ctx) foreach (var doc in ctx.Documents) { - if (!(doc["ExceptionlessClientInfo"] is JObject clientInfo) || !clientInfo.HasValues || clientInfo["InstallDate"] is null) - return; + if (doc is not JsonObject docObj || docObj["ExceptionlessClientInfo"] is not JsonObject { Count: > 0 } clientInfo || clientInfo["InstallDate"] is null) + continue; // This shouldn't hurt using DateTimeOffset to try and parse a date. It insures you won't lose any info. - if (DateTimeOffset.TryParse(clientInfo["InstallDate"]!.ToString(), out var date)) + if (DateTimeOffset.TryParse(clientInfo["InstallDate"]?.ToString(), CultureInfo.InvariantCulture, DateTimeStyles.None, out var date)) { clientInfo.Remove("InstallDate"); - clientInfo.Add("InstallDate", new JValue(date)); + clientInfo.Add("InstallDate", JsonValue.Create(date)); } else { diff --git a/src/Exceptionless.Core/Plugins/EventUpgrader/Default/V1R844_EventUpgrade.cs b/src/Exceptionless.Core/Plugins/EventUpgrader/Default/V1R844_EventUpgrade.cs index d2db4115c7..78d8168ff0 100644 --- a/src/Exceptionless.Core/Plugins/EventUpgrader/Default/V1R844_EventUpgrade.cs +++ b/src/Exceptionless.Core/Plugins/EventUpgrader/Default/V1R844_EventUpgrade.cs @@ -1,6 +1,6 @@ -using Exceptionless.Core.Pipeline; +using System.Text.Json.Nodes; +using Exceptionless.Core.Pipeline; using Microsoft.Extensions.Logging; -using Newtonsoft.Json.Linq; namespace Exceptionless.Core.Plugins.EventUpgrader; @@ -16,26 +16,22 @@ public void Upgrade(EventUpgraderContext ctx) foreach (var doc in ctx.Documents) { + if (doc is not JsonObject docObj || docObj["RequestInfo"] is not JsonObject { Count: > 0 } requestInfo) + continue; - if (!(doc["RequestInfo"] is JObject { HasValues: true } requestInfo)) - return; - - if (requestInfo["Cookies"] is not null && requestInfo["Cookies"]!.HasValues) + if (requestInfo["Cookies"] is JsonObject { Count: > 0 } cookies) { - if (requestInfo["Cookies"] is JObject cookies) - cookies.Remove(""); + cookies.Remove(""); } - if (requestInfo["Form"] is not null && requestInfo["Form"]!.HasValues) + if (requestInfo["Form"] is JsonObject { Count: > 0 } form) { - if (requestInfo["Form"] is JObject form) - form.Remove(""); + form.Remove(""); } - if (requestInfo["QueryString"] is not null && requestInfo["QueryString"]!.HasValues) + if (requestInfo["QueryString"] is JsonObject { Count: > 0 } queryString) { - if (requestInfo["QueryString"] is JObject queryString) - queryString.Remove(""); + queryString.Remove(""); } } } diff --git a/src/Exceptionless.Core/Plugins/EventUpgrader/Default/V1R850_EventUpgrade.cs b/src/Exceptionless.Core/Plugins/EventUpgrader/Default/V1R850_EventUpgrade.cs index 424ef62010..418a27438f 100644 --- a/src/Exceptionless.Core/Plugins/EventUpgrader/Default/V1R850_EventUpgrade.cs +++ b/src/Exceptionless.Core/Plugins/EventUpgrader/Default/V1R850_EventUpgrade.cs @@ -1,7 +1,7 @@ -using Exceptionless.Core.Extensions; +using System.Text.Json.Nodes; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Pipeline; using Microsoft.Extensions.Logging; -using Newtonsoft.Json.Linq; namespace Exceptionless.Core.Plugins.EventUpgrader; @@ -15,12 +15,12 @@ public void Upgrade(EventUpgraderContext ctx) if (ctx.Version > new Version(1, 0, 0, 850)) return; - foreach (var doc in ctx.Documents.OfType()) + foreach (var doc in ctx.Documents.OfType()) { var current = doc; while (current is not null) { - if (doc["ExtendedData"] is JObject extendedData) + if (current["ExtendedData"] is JsonObject extendedData) { if (extendedData["ExtraExceptionProperties"] is not null) extendedData.Rename("ExtraExceptionProperties", "__ExceptionInfo"); @@ -32,7 +32,7 @@ public void Upgrade(EventUpgraderContext ctx) extendedData.Rename("TraceInfo", "TraceLog"); } - current = current["Inner"] as JObject; + current = current["Inner"] as JsonObject; } } } diff --git a/src/Exceptionless.Core/Plugins/EventUpgrader/Default/V2_EventUpgrade.cs b/src/Exceptionless.Core/Plugins/EventUpgrader/Default/V2_EventUpgrade.cs index 5611ccb5b4..ad4fdd49b0 100644 --- a/src/Exceptionless.Core/Plugins/EventUpgrader/Default/V2_EventUpgrade.cs +++ b/src/Exceptionless.Core/Plugins/EventUpgrader/Default/V2_EventUpgrade.cs @@ -1,8 +1,9 @@ -using Exceptionless.Core.Extensions; +using System.Text.Json; +using System.Text.Json.Nodes; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models.Data; using Exceptionless.Core.Pipeline; using Microsoft.Extensions.Logging; -using Newtonsoft.Json.Linq; namespace Exceptionless.Core.Plugins.EventUpgrader; @@ -16,7 +17,7 @@ public void Upgrade(EventUpgraderContext ctx) if (ctx.Version > new Version(2, 0)) return; - foreach (var doc in ctx.Documents.OfType()) + foreach (var doc in ctx.Documents.OfType()) { bool isNotFound = doc.GetPropertyStringValue("Code") == "404"; @@ -36,15 +37,18 @@ public void Upgrade(EventUpgraderContext ctx) doc.Remove("ExceptionlessClientInfo"); if (!doc.RemoveIfNullOrEmpty("Tags")) { - var tags = doc.GetValue("Tags"); - if (tags is not null && tags.Type == JTokenType.Array) + var tags = doc["Tags"]; + if (tags is JsonArray tagsArray) { - foreach (var tag in tags.ToList()) + var tagsToRemove = new List(); + foreach (var tag in tagsArray) { - string t = tag.ToString(); + string? t = tag?.ToString(); if (String.IsNullOrEmpty(t) || t.Length > 255) - tag.Remove(); + tagsToRemove.Add(tag); } + foreach (var tag in tagsToRemove) + tagsArray.Remove(tag); } } @@ -58,7 +62,7 @@ public void Upgrade(EventUpgraderContext ctx) doc.RenameAll("ExtendedData", "Data"); - var extendedData = doc.Property("Data") is not null ? doc.Property("Data")!.Value as JObject : null; + var extendedData = doc["Data"] as JsonObject; if (extendedData is not null) { if (!isNotFound) @@ -73,58 +77,62 @@ public void Upgrade(EventUpgraderContext ctx) if (extendedData?["__ExceptionInfo"] is not null) extendedData.Remove("__ExceptionInfo"); - doc.Add("Type", new JValue("404")); + doc.Add("Type", JsonValue.Create("404")); } else { - var error = new JObject(); + var error = new JsonObject(); if (!doc.RemoveIfNullOrEmpty("Message")) - error.Add("Message", doc["Message"]!.Value()); + { + var messageValue = doc["Message"]?.GetValue(); + if (messageValue is not null) + error.Add("Message", JsonValue.Create(messageValue)); + } error.MoveOrRemoveIfNullOrEmpty(doc, "Code", "Type", "Inner", "StackTrace", "TargetMethod", "Modules"); // Copy the exception info from root extended data to the current errors extended data. if (extendedData?["__ExceptionInfo"] is not null) { - error.Add("Data", new JObject()); - ((JObject)error["Data"]!).MoveOrRemoveIfNullOrEmpty(extendedData, "__ExceptionInfo"); + error.Add("Data", new JsonObject()); + ((JsonObject)error["Data"]!).MoveOrRemoveIfNullOrEmpty(extendedData, "__ExceptionInfo"); } - string? id = doc["Id"]?.Value(); + string? id = doc["Id"]?.GetValue(); RenameAndValidateExtraExceptionProperties(id, error); - var inner = error["Inner"] as JObject; + var inner = error["Inner"] as JsonObject; while (inner is not null) { RenameAndValidateExtraExceptionProperties(id, inner); - inner = inner["Inner"] as JObject; + inner = inner["Inner"] as JsonObject; } - doc.Add("Type", new JValue(isNotFound ? "404" : "error")); + doc.Add("Type", JsonValue.Create(isNotFound ? "404" : "error")); doc.Add("@error", error); } string? emailAddress = doc.GetPropertyStringValueAndRemove("UserEmail"); string? userDescription = doc.GetPropertyStringValueAndRemove("UserDescription"); if (!String.IsNullOrWhiteSpace(emailAddress) && !String.IsNullOrWhiteSpace(userDescription)) - doc.Add("@user_description", JObject.FromObject(new UserDescription(emailAddress, userDescription))); + doc.Add("@user_description", JsonSerializer.SerializeToNode(new UserDescription(emailAddress, userDescription))); string? identity = doc.GetPropertyStringValueAndRemove("UserName"); if (!String.IsNullOrWhiteSpace(identity)) - doc.Add("@user", JObject.FromObject(new UserInfo(identity))); + doc.Add("@user", JsonSerializer.SerializeToNode(new UserInfo(identity))); doc.RemoveAllIfNullOrEmpty("Data", "GenericArguments", "Parameters"); } } - private void RenameAndValidateExtraExceptionProperties(string? id, JObject error) + private void RenameAndValidateExtraExceptionProperties(string? id, JsonObject error) { - var extendedData = error?["Data"] as JObject; + var extendedData = error["Data"] as JsonObject; if (extendedData?["__ExceptionInfo"] is null) return; - string json = extendedData["__ExceptionInfo"]!.ToString(); + string? json = extendedData["__ExceptionInfo"]?.ToString(); extendedData.Remove("__ExceptionInfo"); if (String.IsNullOrWhiteSpace(json)) @@ -136,23 +144,28 @@ private void RenameAndValidateExtraExceptionProperties(string? id, JObject error return; } - var ext = new JObject(); + var ext = new JsonObject(); try { - var extraProperties = JObject.Parse(json); - foreach (var property in extraProperties.Properties()) + var extraProperties = JsonNode.Parse(json) as JsonObject; + if (extraProperties is not null) { - if (property.IsNullOrEmpty()) - continue; - - string dataKey = property.Name; - if (extendedData[dataKey] is not null) - dataKey = "_" + dataKey; + foreach (var property in extraProperties.ToList().Where(p => !p.Value.IsNullOrEmpty())) + { + string dataKey = property.Key; + if (extendedData[dataKey] is not null) + dataKey = "_" + dataKey; - ext.Add(dataKey, property.Value); + // Need to detach the node before adding to another parent + extraProperties.Remove(property.Key); + ext.Add(dataKey, property.Value); + } } } - catch (Exception) { } + catch (JsonException ex) + { + _logger.LogWarning(ex, "Failed to parse __ExceptionInfo JSON for event {Id}: {Message}", id, ex.Message); + } if (ext.IsNullOrEmpty()) return; diff --git a/src/Exceptionless.Core/Plugins/EventUpgrader/EventUpgraderContext.cs b/src/Exceptionless.Core/Plugins/EventUpgrader/EventUpgraderContext.cs index 1d51dd4e3d..70eefc6e3d 100644 --- a/src/Exceptionless.Core/Plugins/EventUpgrader/EventUpgraderContext.cs +++ b/src/Exceptionless.Core/Plugins/EventUpgrader/EventUpgraderContext.cs @@ -1,7 +1,6 @@ -using Exceptionless.Core.Extensions; +using System.Text.Json.Nodes; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Utility; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; namespace Exceptionless.Core.Plugins.EventUpgrader; @@ -12,15 +11,15 @@ public EventUpgraderContext(string json, Version? version = null, bool isMigrati var jsonType = json.GetJsonType(); if (jsonType == JsonType.Object) { - var doc = JsonConvert.DeserializeObject(json); + var doc = JsonNode.Parse(json) as JsonObject; if (doc is not null) - Documents = new JArray(doc); + Documents = new JsonArray(doc); else throw new ArgumentException("Invalid json object specified", nameof(json)); } else if (jsonType == JsonType.Array) { - var docs = JsonConvert.DeserializeObject(json); + var docs = JsonNode.Parse(json) as JsonArray; if (docs is not null) Documents = docs; else @@ -35,21 +34,21 @@ public EventUpgraderContext(string json, Version? version = null, bool isMigrati IsMigration = isMigration; } - public EventUpgraderContext(JObject doc, Version? version = null, bool isMigration = false) + public EventUpgraderContext(JsonObject doc, Version? version = null, bool isMigration = false) { - Documents = new JArray(doc); + Documents = new JsonArray(doc); Version = version; IsMigration = isMigration; } - public EventUpgraderContext(JArray docs, Version? version = null, bool isMigration = false) + public EventUpgraderContext(JsonArray docs, Version? version = null, bool isMigration = false) { Documents = docs; Version = version; IsMigration = isMigration; } - public JArray Documents { get; set; } + public JsonArray Documents { get; set; } public Version? Version { get; set; } public bool IsMigration { get; set; } } diff --git a/src/Exceptionless.Core/Plugins/Formatting/Default/05_ManualStackingFormattingPlugin.cs b/src/Exceptionless.Core/Plugins/Formatting/Default/05_ManualStackingFormattingPlugin.cs index 3f3d70dbf5..9cd8df9aad 100644 --- a/src/Exceptionless.Core/Plugins/Formatting/Default/05_ManualStackingFormattingPlugin.cs +++ b/src/Exceptionless.Core/Plugins/Formatting/Default/05_ManualStackingFormattingPlugin.cs @@ -1,6 +1,6 @@ -using System.Text.Json; -using Exceptionless.Core.Models; +using Exceptionless.Core.Models; using Exceptionless.Core.Pipeline; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.Formatting; @@ -8,11 +8,11 @@ namespace Exceptionless.Core.Plugins.Formatting; [Priority(5)] public sealed class ManualStackingFormattingPlugin : FormattingPluginBase { - public ManualStackingFormattingPlugin(JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(jsonOptions, options, loggerFactory) { } + public ManualStackingFormattingPlugin(ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(serializer, options, loggerFactory) { } public override string? GetStackTitle(PersistentEvent ev) { - var msi = ev.GetManualStackingInfo(_jsonOptions); + var msi = ev.GetManualStackingInfo(_serializer, _logger); return !String.IsNullOrWhiteSpace(msi?.Title) ? msi.Title : null; } } diff --git a/src/Exceptionless.Core/Plugins/Formatting/Default/10_SimpleErrorFormattingPlugin.cs b/src/Exceptionless.Core/Plugins/Formatting/Default/10_SimpleErrorFormattingPlugin.cs index 8b79ac1c28..6dbdfef7a3 100644 --- a/src/Exceptionless.Core/Plugins/Formatting/Default/10_SimpleErrorFormattingPlugin.cs +++ b/src/Exceptionless.Core/Plugins/Formatting/Default/10_SimpleErrorFormattingPlugin.cs @@ -1,7 +1,7 @@ -using System.Text.Json; -using Exceptionless.Core.Extensions; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Pipeline; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.Formatting; @@ -9,7 +9,7 @@ namespace Exceptionless.Core.Plugins.Formatting; [Priority(10)] public sealed class SimpleErrorFormattingPlugin : FormattingPluginBase { - public SimpleErrorFormattingPlugin(JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(jsonOptions, options, loggerFactory) { } + public SimpleErrorFormattingPlugin(ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(serializer, options, loggerFactory) { } private bool ShouldHandle(PersistentEvent ev) { @@ -39,7 +39,7 @@ private bool ShouldHandle(PersistentEvent ev) if (!ShouldHandle(ev)) return null; - var error = ev.GetSimpleError(_jsonOptions); + var error = ev.GetSimpleError(_serializer, _logger); return error?.Message; } @@ -48,12 +48,12 @@ private bool ShouldHandle(PersistentEvent ev) if (!ShouldHandle(ev)) return null; - var error = ev.GetSimpleError(_jsonOptions); + var error = ev.GetSimpleError(_serializer, _logger); if (error is null) return null; var data = new Dictionary { { "Message", ev.Message } }; - AddUserIdentitySummaryData(data, ev.GetUserIdentity(_jsonOptions)); + AddUserIdentitySummaryData(data, ev.GetUserIdentity(_serializer, _logger)); if (!String.IsNullOrEmpty(error.Type)) { @@ -61,7 +61,7 @@ private bool ShouldHandle(PersistentEvent ev) data.Add("TypeFullName", error.Type); } - var requestInfo = ev.GetRequestInfo(_jsonOptions); + var requestInfo = ev.GetRequestInfo(_serializer, _logger); if (!String.IsNullOrEmpty(requestInfo?.Path)) data.Add("Path", requestInfo.Path); @@ -73,7 +73,7 @@ private bool ShouldHandle(PersistentEvent ev) if (!ShouldHandle(ev)) return null; - var error = ev.GetSimpleError(_jsonOptions); + var error = ev.GetSimpleError(_serializer, _logger); if (error is null) return null; @@ -96,7 +96,7 @@ private bool ShouldHandle(PersistentEvent ev) if (!String.IsNullOrEmpty(errorTypeName)) data.Add("Type", errorTypeName); - var requestInfo = ev.GetRequestInfo(_jsonOptions); + var requestInfo = ev.GetRequestInfo(_serializer, _logger); if (requestInfo is not null) data.Add("Url", requestInfo.GetFullPath(true, true, true)); @@ -108,7 +108,7 @@ private bool ShouldHandle(PersistentEvent ev) if (!ShouldHandle(ev)) return null; - var error = ev.GetSimpleError(_jsonOptions); + var error = ev.GetSimpleError(_serializer, _logger); if (error is null) return null; @@ -126,7 +126,7 @@ private bool ShouldHandle(PersistentEvent ev) if (isCritical) notificationType = String.Concat("critical ", notificationType); - var attachment = new SlackMessage.SlackAttachment(ev, _jsonOptions) + var attachment = new SlackMessage.SlackAttachment(ev, _serializer, _logger) { Color = "#BB423F", Fields = diff --git a/src/Exceptionless.Core/Plugins/Formatting/Default/20_ErrorFormattingPlugin.cs b/src/Exceptionless.Core/Plugins/Formatting/Default/20_ErrorFormattingPlugin.cs index 250bdf945c..3084b18778 100644 --- a/src/Exceptionless.Core/Plugins/Formatting/Default/20_ErrorFormattingPlugin.cs +++ b/src/Exceptionless.Core/Plugins/Formatting/Default/20_ErrorFormattingPlugin.cs @@ -1,7 +1,7 @@ -using System.Text.Json; -using Exceptionless.Core.Extensions; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Pipeline; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.Formatting; @@ -9,7 +9,7 @@ namespace Exceptionless.Core.Plugins.Formatting; [Priority(20)] public sealed class ErrorFormattingPlugin : FormattingPluginBase { - public ErrorFormattingPlugin(JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(jsonOptions, options, loggerFactory) { } + public ErrorFormattingPlugin(ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(serializer, options, loggerFactory) { } private bool ShouldHandle(PersistentEvent ev) { @@ -21,7 +21,7 @@ private bool ShouldHandle(PersistentEvent ev) if (!ShouldHandle(ev)) return null; - var error = ev.GetError(_jsonOptions); + var error = ev.GetError(_serializer, _logger); return error?.Message; } @@ -59,12 +59,12 @@ private bool ShouldHandle(PersistentEvent ev) if (!ShouldHandle(ev)) return null; - var stackingTarget = ev.GetStackingTarget(_jsonOptions); + var stackingTarget = ev.GetStackingTarget(_serializer, _logger); if (stackingTarget?.Error is null) return null; var data = new Dictionary { { "Message", ev.Message } }; - AddUserIdentitySummaryData(data, ev.GetUserIdentity(_jsonOptions)); + AddUserIdentitySummaryData(data, ev.GetUserIdentity(_serializer, _logger)); if (!String.IsNullOrEmpty(stackingTarget.Error.Type)) { @@ -78,7 +78,7 @@ private bool ShouldHandle(PersistentEvent ev) data.Add("MethodFullName", stackingTarget.Method.GetFullName()); } - var requestInfo = ev.GetRequestInfo(_jsonOptions); + var requestInfo = ev.GetRequestInfo(_serializer, _logger); if (!String.IsNullOrEmpty(requestInfo?.Path)) data.Add("Path", requestInfo.Path); @@ -90,7 +90,7 @@ private bool ShouldHandle(PersistentEvent ev) if (!ShouldHandle(ev)) return null; - var error = ev.GetError(_jsonOptions); + var error = ev.GetError(_serializer, _logger); var stackingTarget = error?.GetStackingTarget(); if (stackingTarget?.Error is null) return null; @@ -117,7 +117,7 @@ private bool ShouldHandle(PersistentEvent ev) if (stackingTarget.Method?.Name is not null) data.Add("Method", stackingTarget.Method.Name.Truncate(60)); - var requestInfo = ev.GetRequestInfo(_jsonOptions); + var requestInfo = ev.GetRequestInfo(_serializer, _logger); if (requestInfo is not null) data.Add("Url", requestInfo.GetFullPath(true, true, true)); @@ -129,7 +129,7 @@ private bool ShouldHandle(PersistentEvent ev) if (!ShouldHandle(ev)) return null; - var error = ev.GetError(_jsonOptions); + var error = ev.GetError(_serializer, _logger); var stackingTarget = error?.GetStackingTarget(); if (stackingTarget?.Error is null) return null; @@ -148,7 +148,7 @@ private bool ShouldHandle(PersistentEvent ev) if (isCritical) notificationType = String.Concat("critical ", notificationType); - var attachment = new SlackMessage.SlackAttachment(ev, _jsonOptions) + var attachment = new SlackMessage.SlackAttachment(ev, _serializer, _logger) { Color = "#BB423F", Fields = diff --git a/src/Exceptionless.Core/Plugins/Formatting/Default/30_NotFoundFormattingPlugin.cs b/src/Exceptionless.Core/Plugins/Formatting/Default/30_NotFoundFormattingPlugin.cs index c3f602bf24..1ed977c6f0 100644 --- a/src/Exceptionless.Core/Plugins/Formatting/Default/30_NotFoundFormattingPlugin.cs +++ b/src/Exceptionless.Core/Plugins/Formatting/Default/30_NotFoundFormattingPlugin.cs @@ -1,7 +1,7 @@ -using System.Text.Json; -using Exceptionless.Core.Extensions; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Pipeline; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.Formatting; @@ -9,7 +9,7 @@ namespace Exceptionless.Core.Plugins.Formatting; [Priority(30)] public sealed class NotFoundFormattingPlugin : FormattingPluginBase { - public NotFoundFormattingPlugin(JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(jsonOptions, options, loggerFactory) { } + public NotFoundFormattingPlugin(ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(serializer, options, loggerFactory) { } private bool ShouldHandle(PersistentEvent ev) { @@ -38,9 +38,9 @@ private bool ShouldHandle(PersistentEvent ev) return null; var data = new Dictionary { { "Source", ev.Source } }; - AddUserIdentitySummaryData(data, ev.GetUserIdentity(_jsonOptions)); + AddUserIdentitySummaryData(data, ev.GetUserIdentity(_serializer, _logger)); - var ips = ev.GetIpAddresses(_jsonOptions).ToList(); + var ips = ev.GetIpAddresses(_serializer, _logger).ToList(); if (ips.Count > 0) data.Add("IpAddress", ips); @@ -62,7 +62,7 @@ private bool ShouldHandle(PersistentEvent ev) notificationType = String.Concat("Critical ", notificationType.ToLowerInvariant()); string subject = String.Concat(notificationType, ": ", ev.Source).Truncate(120); - var requestInfo = ev.GetRequestInfo(_jsonOptions); + var requestInfo = ev.GetRequestInfo(_serializer, _logger); var data = new Dictionary { { "Url", requestInfo?.GetFullPath(true, true, true) ?? ev.Source?.Truncate(60) } }; @@ -84,8 +84,8 @@ private bool ShouldHandle(PersistentEvent ev) if (isCritical) notificationType = String.Concat("critical ", notificationType); - var requestInfo = ev.GetRequestInfo(_jsonOptions); - var attachment = new SlackMessage.SlackAttachment(ev, _jsonOptions) + var requestInfo = ev.GetRequestInfo(_serializer, _logger); + var attachment = new SlackMessage.SlackAttachment(ev, _serializer, _logger) { Color = "#BB423F", Fields = diff --git a/src/Exceptionless.Core/Plugins/Formatting/Default/40_UsageFormattingPlugin.cs b/src/Exceptionless.Core/Plugins/Formatting/Default/40_UsageFormattingPlugin.cs index 6b02b61052..c74d6d9c86 100644 --- a/src/Exceptionless.Core/Plugins/Formatting/Default/40_UsageFormattingPlugin.cs +++ b/src/Exceptionless.Core/Plugins/Formatting/Default/40_UsageFormattingPlugin.cs @@ -1,7 +1,7 @@ -using System.Text.Json; -using Exceptionless.Core.Extensions; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Pipeline; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.Formatting; @@ -9,7 +9,7 @@ namespace Exceptionless.Core.Plugins.Formatting; [Priority(40)] public sealed class UsageFormattingPlugin : FormattingPluginBase { - public UsageFormattingPlugin(JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(jsonOptions, options, loggerFactory) { } + public UsageFormattingPlugin(ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(serializer, options, loggerFactory) { } private bool ShouldHandle(PersistentEvent ev) { @@ -38,7 +38,7 @@ private bool ShouldHandle(PersistentEvent ev) return null; var data = new Dictionary { { "Source", ev.Source } }; - AddUserIdentitySummaryData(data, ev.GetUserIdentity(_jsonOptions)); + AddUserIdentitySummaryData(data, ev.GetUserIdentity(_serializer, _logger)); return new SummaryData { Id = ev.Id, TemplateKey = "event-feature-summary", Data = data }; } @@ -61,7 +61,7 @@ private bool ShouldHandle(PersistentEvent ev) if (!ShouldHandle(ev)) return null; - var attachment = new SlackMessage.SlackAttachment(ev, _jsonOptions) + var attachment = new SlackMessage.SlackAttachment(ev, _serializer, _logger) { Fields = [ diff --git a/src/Exceptionless.Core/Plugins/Formatting/Default/50_SessionFormattingPlugin.cs b/src/Exceptionless.Core/Plugins/Formatting/Default/50_SessionFormattingPlugin.cs index 1dc5710829..9bba237760 100644 --- a/src/Exceptionless.Core/Plugins/Formatting/Default/50_SessionFormattingPlugin.cs +++ b/src/Exceptionless.Core/Plugins/Formatting/Default/50_SessionFormattingPlugin.cs @@ -1,7 +1,7 @@ -using System.Text.Json; -using Exceptionless.Core.Extensions; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Pipeline; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.Formatting; @@ -9,7 +9,7 @@ namespace Exceptionless.Core.Plugins.Formatting; [Priority(50)] public sealed class SessionFormattingPlugin : FormattingPluginBase { - public SessionFormattingPlugin(JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(jsonOptions, options, loggerFactory) { } + public SessionFormattingPlugin(ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(serializer, options, loggerFactory) { } private bool ShouldHandle(PersistentEvent ev) { @@ -41,7 +41,7 @@ private bool ShouldHandle(PersistentEvent ev) return null; var data = new Dictionary { { "SessionId", ev.GetSessionId() }, { "Type", ev.Type } }; - AddUserIdentitySummaryData(data, ev.GetUserIdentity(_jsonOptions)); + AddUserIdentitySummaryData(data, ev.GetUserIdentity(_serializer, _logger)); if (ev.IsSessionStart()) { diff --git a/src/Exceptionless.Core/Plugins/Formatting/Default/60_LogFormattingPlugin.cs b/src/Exceptionless.Core/Plugins/Formatting/Default/60_LogFormattingPlugin.cs index 3eca0c290c..f00ffc6278 100644 --- a/src/Exceptionless.Core/Plugins/Formatting/Default/60_LogFormattingPlugin.cs +++ b/src/Exceptionless.Core/Plugins/Formatting/Default/60_LogFormattingPlugin.cs @@ -1,7 +1,7 @@ -using System.Text.Json; -using Exceptionless.Core.Extensions; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Pipeline; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.Formatting; @@ -9,7 +9,7 @@ namespace Exceptionless.Core.Plugins.Formatting; [Priority(60)] public sealed class LogFormattingPlugin : FormattingPluginBase { - public LogFormattingPlugin(JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(jsonOptions, options, loggerFactory) { } + public LogFormattingPlugin(ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(serializer, options, loggerFactory) { } private bool ShouldHandle(PersistentEvent ev) { @@ -50,7 +50,7 @@ private bool ShouldHandle(PersistentEvent ev) return null; var data = new Dictionary { { "Message", ev.Message } }; - AddUserIdentitySummaryData(data, ev.GetUserIdentity(_jsonOptions)); + AddUserIdentitySummaryData(data, ev.GetUserIdentity(_serializer, _logger)); if (!String.IsNullOrWhiteSpace(ev.Source)) { @@ -92,7 +92,7 @@ private bool ShouldHandle(PersistentEvent ev) if (!String.IsNullOrEmpty(level)) data.Add("Level", level.Truncate(60)); - var requestInfo = ev.GetRequestInfo(_jsonOptions); + var requestInfo = ev.GetRequestInfo(_serializer, _logger); if (requestInfo is not null) data.Add("Url", requestInfo.GetFullPath(true, true, true)); @@ -114,7 +114,7 @@ private bool ShouldHandle(PersistentEvent ev) notificationType = String.Concat("critical ", notificationType); string source = !String.IsNullOrEmpty(ev.Source) ? ev.Source : "(Global)"; - var attachment = new SlackMessage.SlackAttachment(ev, _jsonOptions) + var attachment = new SlackMessage.SlackAttachment(ev, _serializer, _logger) { Fields = [ @@ -149,7 +149,7 @@ private bool ShouldHandle(PersistentEvent ev) attachment.Fields.Add(new SlackMessage.SlackAttachmentFields { Title = "Level", Value = level.Truncate(60) }); } - var requestInfo = ev.GetRequestInfo(_jsonOptions); + var requestInfo = ev.GetRequestInfo(_serializer, _logger); if (requestInfo is not null) attachment.Fields.Add(new SlackMessage.SlackAttachmentFields { Title = "Url", Value = requestInfo.GetFullPath(true, true, true) }); diff --git a/src/Exceptionless.Core/Plugins/Formatting/Default/99_DefaultFormattingPlugin.cs b/src/Exceptionless.Core/Plugins/Formatting/Default/99_DefaultFormattingPlugin.cs index 39ec6593e9..db42da417b 100644 --- a/src/Exceptionless.Core/Plugins/Formatting/Default/99_DefaultFormattingPlugin.cs +++ b/src/Exceptionless.Core/Plugins/Formatting/Default/99_DefaultFormattingPlugin.cs @@ -1,7 +1,7 @@ -using System.Text.Json; -using Exceptionless.Core.Extensions; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Pipeline; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.Formatting; @@ -9,7 +9,7 @@ namespace Exceptionless.Core.Plugins.Formatting; [Priority(99)] public sealed class DefaultFormattingPlugin : FormattingPluginBase { - public DefaultFormattingPlugin(JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(jsonOptions, options, loggerFactory) { } + public DefaultFormattingPlugin(ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(serializer, options, loggerFactory) { } public override string GetStackTitle(PersistentEvent ev) { @@ -37,7 +37,7 @@ public override SummaryData GetEventSummaryData(PersistentEvent ev) { "Type", ev.Type } }; - AddUserIdentitySummaryData(data, ev.GetUserIdentity(_jsonOptions)); + AddUserIdentitySummaryData(data, ev.GetUserIdentity(_serializer, _logger)); return new SummaryData { Id = ev.Id, TemplateKey = "event-summary", Data = data }; } @@ -68,7 +68,7 @@ public override MailMessageData GetEventNotificationMailMessageData(PersistentEv if (!String.IsNullOrEmpty(ev.Source)) data.Add("Source", ev.Source.Truncate(60)); - var requestInfo = ev.GetRequestInfo(_jsonOptions); + var requestInfo = ev.GetRequestInfo(_serializer, _logger); if (requestInfo is not null) data.Add("Url", requestInfo.GetFullPath(true, true, true)); @@ -90,7 +90,7 @@ public override SlackMessage GetSlackEventNotification(PersistentEvent ev, Proje if (isCritical) notificationType = String.Concat("Critical ", notificationType.ToLowerInvariant()); - var attachment = new SlackMessage.SlackAttachment(ev, _jsonOptions); + var attachment = new SlackMessage.SlackAttachment(ev, _serializer, _logger); if (!String.IsNullOrEmpty(ev.Message)) attachment.Fields.Add(new SlackMessage.SlackAttachmentFields { Title = "Message", Value = ev.Message.Truncate(60) }); diff --git a/src/Exceptionless.Core/Plugins/Formatting/FormattingPluginBase.cs b/src/Exceptionless.Core/Plugins/Formatting/FormattingPluginBase.cs index 926a83eb95..1b4960aa37 100644 --- a/src/Exceptionless.Core/Plugins/Formatting/FormattingPluginBase.cs +++ b/src/Exceptionless.Core/Plugins/Formatting/FormattingPluginBase.cs @@ -1,18 +1,18 @@ -using System.Text.Json; -using Exceptionless.Core.Extensions; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Models.Data; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.Formatting; public abstract class FormattingPluginBase : PluginBase, IFormattingPlugin { - protected readonly JsonSerializerOptions _jsonOptions; + protected readonly ITextSerializer _serializer; - public FormattingPluginBase(JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) + public FormattingPluginBase(ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) { - _jsonOptions = jsonOptions; + _serializer = serializer; } public virtual SummaryData? GetStackSummaryData(Stack stack) @@ -42,7 +42,7 @@ public FormattingPluginBase(JsonSerializerOptions jsonOptions, AppOptions option protected void AddDefaultSlackFields(PersistentEvent ev, List attachmentFields, bool includeUrl = true) { - var requestInfo = ev.GetRequestInfo(_jsonOptions); + var requestInfo = ev.GetRequestInfo(_serializer, _logger); if (requestInfo is not null && includeUrl) attachmentFields.Add(new SlackMessage.SlackAttachmentFields { Title = "Url", Value = requestInfo.GetFullPath(true, true, true) }); diff --git a/src/Exceptionless.Core/Plugins/WebHook/Default/005_SlackPlugin.cs b/src/Exceptionless.Core/Plugins/WebHook/Default/005_SlackPlugin.cs index 45c031a826..81a1557863 100644 --- a/src/Exceptionless.Core/Plugins/WebHook/Default/005_SlackPlugin.cs +++ b/src/Exceptionless.Core/Plugins/WebHook/Default/005_SlackPlugin.cs @@ -1,6 +1,6 @@ -using System.Text.Json; -using Exceptionless.Core.Pipeline; +using Exceptionless.Core.Pipeline; using Exceptionless.Core.Plugins.Formatting; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.WebHook; @@ -9,12 +9,12 @@ namespace Exceptionless.Core.Plugins.WebHook; public sealed class SlackPlugin : WebHookDataPluginBase { private readonly FormattingPluginManager _pluginManager; - private readonly JsonSerializerOptions _jsonOptions; + private readonly ITextSerializer _serializer; - public SlackPlugin(FormattingPluginManager pluginManager, JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) + public SlackPlugin(FormattingPluginManager pluginManager, ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) { _pluginManager = pluginManager; - _jsonOptions = jsonOptions; + _serializer = serializer; } public override Task CreateFromEventAsync(WebHookDataContext ctx) @@ -22,7 +22,7 @@ public SlackPlugin(FormattingPluginManager pluginManager, JsonSerializerOptions if (String.IsNullOrEmpty(ctx.WebHook.Url) || !ctx.WebHook.Url.EndsWith("/slack")) return Task.FromResult(null); - var error = ctx.Event?.GetError(_jsonOptions); + var error = ctx.Event?.GetError(_serializer, _logger); if (error is null) { ctx.IsCancelled = true; diff --git a/src/Exceptionless.Core/Plugins/WebHook/Default/010_VersionOnePlugin.cs b/src/Exceptionless.Core/Plugins/WebHook/Default/010_VersionOnePlugin.cs index c845dcd872..ae05ea24c1 100644 --- a/src/Exceptionless.Core/Plugins/WebHook/Default/010_VersionOnePlugin.cs +++ b/src/Exceptionless.Core/Plugins/WebHook/Default/010_VersionOnePlugin.cs @@ -1,7 +1,10 @@ using System.Text.Json; +using System.Text.Json.Serialization; +using System.Text.Json.Serialization.Metadata; using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Pipeline; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.WebHook; @@ -9,11 +12,20 @@ namespace Exceptionless.Core.Plugins.WebHook; [Priority(10)] public sealed class VersionOnePlugin : WebHookDataPluginBase { - private readonly JsonSerializerOptions _jsonOptions; + private readonly ITextSerializer _serializer; - public VersionOnePlugin(JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) + public VersionOnePlugin(ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) { - _jsonOptions = jsonOptions; + _serializer = serializer; + } + + public static JsonSerializerOptions CreateJsonSerializerOptions(JsonSerializerOptions jsonOptions) + { + return new JsonSerializerOptions(jsonOptions) + { + DefaultIgnoreCondition = JsonIgnoreCondition.Never, + TypeInfoResolver = new DefaultJsonTypeInfoResolver() + }; } public override Task CreateFromEventAsync(WebHookDataContext ctx) @@ -21,13 +33,13 @@ public VersionOnePlugin(JsonSerializerOptions jsonOptions, AppOptions options, I if (!String.Equals(ctx.WebHook.Version, Models.WebHook.KnownVersions.Version1)) return Task.FromResult(null); - var error = ctx.Event?.GetError(_jsonOptions); + var error = ctx.Event?.GetError(_serializer, _logger); if (error is null) return Task.FromResult(null); var ev = ctx.Event!; - var requestInfo = ev.GetRequestInfo(_jsonOptions); - var environmentInfo = ev.GetEnvironmentInfo(_jsonOptions); + var requestInfo = ev.GetRequestInfo(_serializer, _logger); + var environmentInfo = ev.GetEnvironmentInfo(_serializer, _logger); return Task.FromResult(new VersionOneWebHookEvent(_options.BaseURL) { @@ -71,9 +83,9 @@ public VersionOnePlugin(JsonSerializerOptions jsonOptions, AppOptions options, I Title = ctx.Stack.Title, Description = ctx.Stack.Description, Tags = ctx.Stack.Tags, - RequestPath = ctx.Stack.SignatureInfo.ContainsKey("Path") ? ctx.Stack.SignatureInfo["Path"] : null, - Type = ctx.Stack.SignatureInfo.ContainsKey("ExceptionType") ? ctx.Stack.SignatureInfo["ExceptionType"] : null, - TargetMethod = ctx.Stack.SignatureInfo.ContainsKey("Method") ? ctx.Stack.SignatureInfo["Method"] : null, + RequestPath = ctx.Stack.SignatureInfo.TryGetValue("Path", out string? path) ? path : null, + Type = ctx.Stack.SignatureInfo.TryGetValue("ExceptionType", out string? exceptionType) ? exceptionType : null, + TargetMethod = ctx.Stack.SignatureInfo.TryGetValue("Method", out string? method) ? method : null, ProjectId = ctx.Stack.ProjectId, ProjectName = ctx.Project.Name, OrganizationId = ctx.Stack.OrganizationId, @@ -83,7 +95,7 @@ public VersionOnePlugin(JsonSerializerOptions jsonOptions, AppOptions options, I LastOccurrence = ctx.Stack.LastOccurrence, DateFixed = ctx.Stack.DateFixed, IsRegression = ctx.Stack.Status == StackStatus.Regressed, - IsCritical = ctx.Stack.OccurrencesAreCritical || ctx.Stack.Tags is not null && ctx.Stack.Tags.Contains("Critical"), + IsCritical = ctx.Stack.OccurrencesAreCritical || ctx.Stack.Tags is not null && ctx.Stack.Tags.Contains(Event.KnownTags.Critical), FixedInVersion = ctx.Stack.FixedInVersion }); } @@ -97,34 +109,62 @@ public VersionOneWebHookEvent(string baseUrl) _baseUrl = baseUrl; } + [JsonPropertyName(nameof(Id))] public string Id { get; init; } = null!; + [JsonPropertyName(nameof(Url))] public string Url => String.Concat(_baseUrl, "/event/", Id); + [JsonPropertyName(nameof(OccurrenceDate))] public DateTimeOffset OccurrenceDate { get; init; } + [JsonPropertyName(nameof(Tags))] public TagSet? Tags { get; init; } = null!; + [JsonPropertyName(nameof(MachineName))] public string? MachineName { get; init; } + [JsonPropertyName(nameof(RequestPath))] public string? RequestPath { get; init; } + [JsonPropertyName(nameof(IpAddress))] public string? IpAddress { get; init; } + [JsonPropertyName(nameof(Message))] public string? Message { get; init; } = null!; + [JsonPropertyName(nameof(Type))] public string? Type { get; init; } = null!; + [JsonPropertyName(nameof(Code))] public string? Code { get; init; } = null!; + [JsonPropertyName(nameof(TargetMethod))] public string? TargetMethod { get; init; } + [JsonPropertyName(nameof(ProjectId))] public string ProjectId { get; init; } = null!; + [JsonPropertyName(nameof(ProjectName))] public string ProjectName { get; init; } = null!; + [JsonPropertyName(nameof(OrganizationId))] public string OrganizationId { get; init; } = null!; + [JsonPropertyName(nameof(OrganizationName))] public string OrganizationName { get; init; } = null!; + [JsonPropertyName(nameof(ErrorStackId))] public string ErrorStackId { get; init; } = null!; + [JsonPropertyName(nameof(ErrorStackStatus))] public StackStatus ErrorStackStatus { get; init; } + [JsonPropertyName(nameof(ErrorStackUrl))] public string ErrorStackUrl => String.Concat(_baseUrl, "/stack/", ErrorStackId); + [JsonPropertyName(nameof(ErrorStackTitle))] public string ErrorStackTitle { get; init; } = null!; + [JsonPropertyName(nameof(ErrorStackDescription))] public string? ErrorStackDescription { get; init; } = null!; + [JsonPropertyName(nameof(ErrorStackTags))] public TagSet ErrorStackTags { get; init; } = null!; + [JsonPropertyName(nameof(TotalOccurrences))] public int TotalOccurrences { get; init; } + [JsonPropertyName(nameof(FirstOccurrence))] public DateTime FirstOccurrence { get; init; } + [JsonPropertyName(nameof(LastOccurrence))] public DateTime LastOccurrence { get; init; } + [JsonPropertyName(nameof(DateFixed))] public DateTime? DateFixed { get; init; } + [JsonPropertyName(nameof(IsNew))] public bool IsNew { get; init; } + [JsonPropertyName(nameof(IsRegression))] public bool IsRegression { get; init; } - public bool IsCritical => Tags is not null && Tags.Contains("Critical"); + [JsonPropertyName(nameof(IsCritical))] + public bool IsCritical => Tags is not null && Tags.Contains(Event.KnownTags.Critical); } public record VersionOneWebHookStack @@ -136,26 +176,45 @@ public VersionOneWebHookStack(string baseUrl) _baseUrl = baseUrl; } + [JsonPropertyName(nameof(Id))] public string Id { get; init; } = null!; + [JsonPropertyName(nameof(Status))] public StackStatus Status { get; init; } + [JsonPropertyName(nameof(Url))] public string Url => String.Concat(_baseUrl, "/stack/", Id); + [JsonPropertyName(nameof(Title))] public string Title { get; init; } = null!; + [JsonPropertyName(nameof(Description))] public string? Description { get; init; } = null!; - + [JsonPropertyName(nameof(Tags))] public TagSet Tags { get; init; } = null!; + [JsonPropertyName(nameof(RequestPath))] public string? RequestPath { get; init; } + [JsonPropertyName(nameof(Type))] public string? Type { get; init; } + [JsonPropertyName(nameof(TargetMethod))] public string? TargetMethod { get; init; } + [JsonPropertyName(nameof(ProjectId))] public string ProjectId { get; init; } = null!; + [JsonPropertyName(nameof(ProjectName))] public string ProjectName { get; init; } = null!; + [JsonPropertyName(nameof(OrganizationId))] public string OrganizationId { get; init; } = null!; + [JsonPropertyName(nameof(OrganizationName))] public string OrganizationName { get; init; } = null!; + [JsonPropertyName(nameof(TotalOccurrences))] public int TotalOccurrences { get; init; } + [JsonPropertyName(nameof(FirstOccurrence))] public DateTime FirstOccurrence { get; init; } + [JsonPropertyName(nameof(LastOccurrence))] public DateTime LastOccurrence { get; init; } + [JsonPropertyName(nameof(DateFixed))] public DateTime? DateFixed { get; init; } + [JsonPropertyName(nameof(FixedInVersion))] public string? FixedInVersion { get; init; } + [JsonPropertyName(nameof(IsRegression))] public bool IsRegression { get; init; } + [JsonPropertyName(nameof(IsCritical))] public bool IsCritical { get; init; } } } diff --git a/src/Exceptionless.Core/Repositories/Configuration/ExceptionlessElasticConfiguration.cs b/src/Exceptionless.Core/Repositories/Configuration/ExceptionlessElasticConfiguration.cs index 2158adeade..2768756410 100644 --- a/src/Exceptionless.Core/Repositories/Configuration/ExceptionlessElasticConfiguration.cs +++ b/src/Exceptionless.Core/Repositories/Configuration/ExceptionlessElasticConfiguration.cs @@ -1,4 +1,6 @@ -using Elasticsearch.Net; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Serialization; +using Elastic.Transport; using Exceptionless.Core.Configuration; using Exceptionless.Core.Extensions; using Exceptionless.Core.Repositories.Queries; @@ -11,32 +13,30 @@ using Foundatio.Repositories.Elasticsearch; using Foundatio.Repositories.Elasticsearch.Configuration; using Foundatio.Repositories.Elasticsearch.Queries.Builders; +using Foundatio.Repositories.Serialization; using Foundatio.Resilience; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; -using Nest; -using Newtonsoft.Json; namespace Exceptionless.Core.Repositories.Configuration; public sealed class ExceptionlessElasticConfiguration : ElasticConfiguration, IStartupAction { private readonly AppOptions _appOptions; - private readonly JsonSerializerSettings _serializerSettings; public ExceptionlessElasticConfiguration( AppOptions appOptions, IQueue workItemQueue, - JsonSerializerSettings serializerSettings, ICacheClient cacheClient, IMessageBus messageBus, IServiceProvider serviceProvider, + ITextSerializer serializer, TimeProvider timeProvider, IResiliencePolicyProvider resiliencePolicyProvider, ILoggerFactory loggerFactory - ) : base(workItemQueue, cacheClient, messageBus, timeProvider, resiliencePolicyProvider, loggerFactory) + ) : base(workItemQueue, cacheClient, messageBus, serializer, timeProvider, resiliencePolicyProvider, loggerFactory) { _appOptions = appOptions; - _serializerSettings = serializerSettings; _logger.LogInformation("All new indexes will be created with {ElasticsearchNumberOfShards} Shards and {ElasticsearchNumberOfReplicas} Replicas", _appOptions.ElasticsearchOptions.NumberOfShards, _appOptions.ElasticsearchOptions.NumberOfReplicas); AddIndex(Stacks = new StackIndex(this)); @@ -77,35 +77,63 @@ public override void ConfigureGlobalQueryBuilders(ElasticQueryBuilder builder) public UserIndex Users { get; } public WebHookIndex WebHooks { get; } - protected override IElasticClient CreateElasticClient() + protected override ElasticsearchClient CreateElasticClient() { var connectionPool = CreateConnectionPool(); - var settings = new ConnectionSettings(connectionPool, (serializer, values) => new ElasticJsonNetSerializer(serializer, values, _serializerSettings)); + + // Settings are intentionally not disposed: they're owned by the ElasticsearchClient for the + // app's lifetime. The configuration is registered as a singleton in DI, so both the settings + // and client live until process exit. + var settings = new ElasticsearchClientSettings( + connectionPool, + sourceSerializer: (_, clientSettings) => + new DefaultSourceSerializer(clientSettings, options => + { + // Base defaults from Foundatio + app-specific overrides + options.ConfigureFoundatioRepositoryDefaults(); + options.ConfigureExceptionlessDefaults(); + + // ES-specific overrides (legacy data compatibility) + options.RespectNullableAnnotations = false; + + // ES needs integers as long and our custom ObjectToInferredTypesConverter. + // Remove any JsonStringEnumConverter (most enums store as integers in ES; + // StackStatus has a type-level [JsonConverter] that takes precedence). + // Remove existing ObjectToInferredTypesConverter instances and insert ours at position 0. + for (int i = options.Converters.Count - 1; i >= 0; i--) + { + if (options.Converters[i] is System.Text.Json.Serialization.JsonStringEnumConverter + or Exceptionless.Core.Serialization.ObjectToInferredTypesConverter + or Foundatio.Repositories.Serialization.ObjectToInferredTypesConverter) + options.Converters.RemoveAt(i); + } + options.Converters.Insert(0, new Exceptionless.Core.Serialization.ObjectToInferredTypesConverter(preferInt64: true)); + })); ConfigureSettings(settings); foreach (var index in Indexes) index.ConfigureSettings(settings); if (!String.IsNullOrEmpty(_appOptions.ElasticsearchOptions.UserName) && !String.IsNullOrEmpty(_appOptions.ElasticsearchOptions.Password)) - settings.BasicAuthentication(_appOptions.ElasticsearchOptions.UserName, _appOptions.ElasticsearchOptions.Password); + settings.Authentication(new BasicAuthentication(_appOptions.ElasticsearchOptions.UserName, _appOptions.ElasticsearchOptions.Password)); - var client = new ElasticClient(settings); + var client = new ElasticsearchClient(settings); return client; } - protected override IConnectionPool CreateConnectionPool() + protected override NodePool CreateConnectionPool() { - var serverUris = Options?.ServerUrl.Split(',').Select(url => new Uri(url)); - return new StaticConnectionPool(serverUris); + var serverUris = Options.ServerUrl?.Split(',').Select(url => new Uri(url)) + ?? throw new InvalidOperationException("ElasticsearchOptions.ServerUrl is not configured."); + return new StaticNodePool(serverUris); } - protected override void ConfigureSettings(ConnectionSettings settings) + protected override void ConfigureSettings(ElasticsearchClientSettings settings) { if (_appOptions.AppMode == AppMode.Development) settings.EnableDebugMode(); - settings.ServerCertificateValidationCallback(CertificateValidations.AllowAll); - settings.EnableApiVersioningHeader(); + settings.ServerCertificateValidationCallback((_, _, _, _) => true); settings.DisableDirectStreaming(); settings.EnableTcpKeepAlive(TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(2)); settings.DefaultFieldNameInferrer(p => p.ToLowerUnderscoredWords()); diff --git a/src/Exceptionless.Core/Repositories/Configuration/Indexes/EventIndex.cs b/src/Exceptionless.Core/Repositories/Configuration/Indexes/EventIndex.cs index 268b6ddc6f..91cc24bc26 100644 --- a/src/Exceptionless.Core/Repositories/Configuration/Indexes/EventIndex.cs +++ b/src/Exceptionless.Core/Repositories/Configuration/Indexes/EventIndex.cs @@ -1,7 +1,15 @@ +using System.Linq.Expressions; +using System.Reflection; +using System.Text.Json; +using System.Text.Json.Serialization.Metadata; +using Elastic.Clients.Elasticsearch.IndexManagement; +using Elastic.Clients.Elasticsearch.Mapping; using Exceptionless.Core.Configuration; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Models.Data; using Exceptionless.Core.Repositories.Queries; +using Exceptionless.Core.Serialization; using Foundatio.Caching; using Foundatio.Parsers.ElasticQueries; using Foundatio.Parsers.ElasticQueries.Extensions; @@ -10,7 +18,6 @@ using Foundatio.Repositories.Elasticsearch.Queries.Builders; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; -using Nest; namespace Exceptionless.Core.Repositories.Configuration; @@ -42,88 +49,85 @@ protected override void ConfigureQueryBuilder(ElasticQueryBuilder builder) builder.RegisterBefore(new EventStackFilterQueryBuilder(stacksRepository, cacheClient, _configuration.LoggerFactory)); } - public override TypeMappingDescriptor ConfigureIndexMapping(TypeMappingDescriptor map) + public override void ConfigureIndexMapping(TypeMappingDescriptor map) { - var mapping = map - .Dynamic(false) + map + .Dynamic(DynamicMapping.False) .DynamicTemplates(dt => dt - .DynamicTemplate("idx_bool", t => t.Match("*-b").Mapping(m => m.Boolean(s => s))) - .DynamicTemplate("idx_date", t => t.Match("*-d").Mapping(m => m.Date(s => s))) - .DynamicTemplate("idx_number", t => t.Match("*-n").Mapping(m => m.Number(s => s.Type(NumberType.Double)))) - .DynamicTemplate("idx_reference", t => t.Match("*-r").Mapping(m => m.Keyword(s => s.IgnoreAbove(256)))) - .DynamicTemplate("idx_string", t => t.Match("*-s").Mapping(m => m.Keyword(s => s.IgnoreAbove(1024))))) + .Add("idx_bool", t => t.Match("*-b").Mapping(m => m.Boolean(s => { }))) + .Add("idx_date", t => t.Match("*-d").Mapping(m => m.Date(s => { }))) + .Add("idx_number", t => t.Match("*-n").Mapping(m => m.DoubleNumber(s => { }))) + .Add("idx_reference", t => t.Match("*-r").Mapping(m => m.Keyword(s => s.IgnoreAbove(256)))) + .Add("idx_string", t => t.Match("*-s").Mapping(m => m.Keyword(s => s.IgnoreAbove(1024))))) .Properties(p => p .SetupDefaults() - .Keyword(f => f.Name(e => e.Id)) - .Keyword(f => f.Name(e => e.OrganizationId)) - .FieldAlias(a => a.Name(Alias.OrganizationId).Path(f => f.OrganizationId)) - .Keyword(f => f.Name(e => e.ProjectId)) - .FieldAlias(a => a.Name(Alias.ProjectId).Path(f => f.ProjectId)) - .Keyword(f => f.Name(e => e.StackId)) - .FieldAlias(a => a.Name(Alias.StackId).Path(f => f.StackId)) - .Keyword(f => f.Name(e => e.ReferenceId)) - .FieldAlias(a => a.Name(Alias.ReferenceId).Path(f => f.ReferenceId)) - .Text(f => f.Name(e => e.Type).Analyzer(LOWER_KEYWORD_ANALYZER).AddKeywordField()) - .Text(f => f.Name(e => e.Source).Analyzer(STANDARDPLUS_ANALYZER).SearchAnalyzer(WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField()) - .Date(f => f.Name(e => e.Date)) - .Text(f => f.Name(e => e.Message)) - .Text(f => f.Name(e => e.Tags).Analyzer(LOWER_KEYWORD_ANALYZER).AddKeywordField()) - .FieldAlias(a => a.Name(Alias.Tags).Path(f => f.Tags)) - .GeoPoint(f => f.Name(e => e.Geo)) - .Scalar(f => f.Value) - .Scalar(f => f.Count) - .Boolean(f => f.Name(e => e.IsFirstOccurrence)) - .FieldAlias(a => a.Name(Alias.IsFirstOccurrence).Path(f => f.IsFirstOccurrence)) - .Object(f => f.Name(e => e.Idx).Dynamic()) - .Object(f => f.Name(e => e.Data).Properties(p2 => p2 - .AddVersionMapping() - .AddLevelMapping() - .AddSubmissionMethodMapping() - .AddSubmissionClientMapping() - .AddLocationMapping() - .AddRequestInfoMapping() - .AddErrorMapping() - .AddSimpleErrorMapping() - .AddEnvironmentInfoMapping() - .AddUserDescriptionMapping() - .AddUserInfoMapping())) + .Keyword(e => e.OrganizationId) + .FieldAlias(Alias.OrganizationId, a => a.Path(f => f.OrganizationId)) + .Keyword(e => e.ProjectId) + .FieldAlias(Alias.ProjectId, a => a.Path(f => f.ProjectId)) + .Keyword(e => e.StackId) + .FieldAlias(Alias.StackId, a => a.Path(f => f.StackId)) + .Keyword(e => e.ReferenceId) + .FieldAlias(Alias.ReferenceId, a => a.Path(f => f.ReferenceId)) + .Text(e => e.Type, t => t.Analyzer(LOWER_KEYWORD_ANALYZER).AddKeywordField()) + .Text(e => e.Source, t => t.Analyzer(STANDARDPLUS_ANALYZER).SearchAnalyzer(WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField()) + .Date(e => e.Date) + .Text(e => e.Message) + .Text(e => e.Tags, t => t.Analyzer(LOWER_KEYWORD_ANALYZER).AddKeywordField()) + .FieldAlias(Alias.Tags, a => a.Path(f => f.Tags)) + .GeoPoint(e => e.Geo) + .DoubleNumber(e => e.Value) + .IntegerNumber(e => e.Count) + .Boolean(e => e.IsFirstOccurrence) + .FieldAlias(Alias.IsFirstOccurrence, a => a.Path(f => f.IsFirstOccurrence)) + .Object(e => e.Idx, o => o.Dynamic(DynamicMapping.True)) + .Object(e => e.Data, o => o.Properties(p2 => p2 + .AddVersionMapping() + .AddLevelMapping() + .AddSubmissionMethodMapping() + .AddSubmissionClientMapping() + .AddLocationMapping() + .AddRequestInfoMapping() + .AddErrorMapping() + .AddSimpleErrorMapping() + .AddEnvironmentInfoMapping() + .AddUserDescriptionMapping() + .AddUserInfoMapping())) .AddCopyToMappings() .AddDataDictionaryAliases() ); if (Options is not null && Options.EnableMapperSizePlugin) - return mapping.SizeField(s => s.Enabled()); - - return mapping; + map.Size(s => s.Enabled(true)); } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - return base.ConfigureIndex(idx.Settings(s => s - .Analysis(BuildAnalysis) + base.ConfigureIndex(idx); + idx.Settings(s => s + .Analysis(a => BuildAnalysis(a)) .NumberOfShards(_configuration.Options.NumberOfShards) .NumberOfReplicas(_configuration.Options.NumberOfReplicas) - .Setting("index.mapping.total_fields.limit", _configuration.Options.FieldsLimit) - .Setting("index.mapping.ignore_malformed", true) - .Priority(1))); + .AddOtherSetting("index.mapping.total_fields.limit", _configuration.Options.FieldsLimit) + .AddOtherSetting("index.mapping.ignore_malformed", true) + .Priority(1)); } public override async Task ConfigureAsync() { const string pipeline = "events-pipeline"; - var response = await Configuration.Client.Ingest.PutPipelineAsync(pipeline, d => d.Processors(p => p - .Script(s => new ScriptProcessor - { - Source = FLATTEN_ERRORS_SCRIPT.Replace("\r", String.Empty).Replace("\n", String.Empty).Replace(" ", " ") - }))); + var response = await Configuration.Client.Ingest.PutPipelineAsync(pipeline, d => d + .Processors(p => p.Script(s => s + .Source(FLATTEN_ERRORS_SCRIPT.TrimScript())))); var logger = Configuration.LoggerFactory.CreateLogger(); logger.LogRequest(response); - if (!response.IsValid) + if (!response.IsValidResponse) { - logger.LogError(response.OriginalException, "Error creating the pipeline {Pipeline}: {Message}", pipeline, response.GetErrorMessage()); - throw new ApplicationException($"Error creating the pipeline {pipeline}: {response.GetErrorMessage()}", response.OriginalException); + string errorMessage = response.DebugInformation; + logger.LogError(response.ApiCallDetails.OriginalException, "Error creating the pipeline {Pipeline}: {Message}", pipeline, errorMessage); + throw new ApplicationException($"Error creating the pipeline {pipeline}: {errorMessage}", response.ApiCallDetails.OriginalException); } await base.ConfigureAsync(); @@ -137,65 +141,65 @@ protected override void ConfigureQueryParser(ElasticQueryParserConfiguration con "source", "message", "tags", - "path", - "error.code", - "error.type", - "error.targettype", - "error.targetmethod", - $"data.{Event.KnownDataKeys.UserDescription}.description", - $"data.{Event.KnownDataKeys.UserDescription}.email_address", - $"data.{Event.KnownDataKeys.UserInfo}.identity", - $"data.{Event.KnownDataKeys.UserInfo}.name" + Alias.RequestPath, + Alias.ErrorCode, + Alias.ErrorType, + Alias.ErrorTargetType, + Alias.ErrorTargetMethod, + EventIndexExtensions.DataPath(Event.KnownDataKeys.UserDescription, u => u.Description), + EventIndexExtensions.DataPath(Event.KnownDataKeys.UserDescription, u => u.EmailAddress), + EventIndexExtensions.DataPath(Event.KnownDataKeys.UserInfo, u => u.Identity), + EventIndexExtensions.DataPath(Event.KnownDataKeys.UserInfo, u => u.Name) ]) .AddQueryVisitor(new EventFieldsQueryVisitor()) .UseFieldMap(new Dictionary { - { Alias.BrowserVersion, $"data.{Event.KnownDataKeys.RequestInfo}.data.{RequestInfo.KnownDataKeys.BrowserVersion}" }, - { Alias.BrowserMajorVersion, $"data.{Event.KnownDataKeys.RequestInfo}.data.{RequestInfo.KnownDataKeys.BrowserMajorVersion}" }, - { Alias.User, $"data.{Event.KnownDataKeys.UserInfo}.identity" }, - { Alias.UserName, $"data.{Event.KnownDataKeys.UserInfo}.name" }, - { Alias.UserEmail, $"data.{Event.KnownDataKeys.UserInfo}.identity" }, - { Alias.UserDescription, $"data.{Event.KnownDataKeys.UserDescription}.description" }, - { Alias.OperatingSystemVersion, $"data.{Event.KnownDataKeys.RequestInfo}.data.{RequestInfo.KnownDataKeys.OSVersion}" }, - { Alias.OperatingSystemMajorVersion, $"data.{Event.KnownDataKeys.RequestInfo}.data.{RequestInfo.KnownDataKeys.OSMajorVersion}" } + { Alias.BrowserVersion, EventIndexExtensions.DataDictionaryPath(Event.KnownDataKeys.RequestInfo, r => r.Data, RequestInfo.KnownDataKeys.BrowserVersion) }, + { Alias.BrowserMajorVersion, EventIndexExtensions.DataDictionaryPath(Event.KnownDataKeys.RequestInfo, r => r.Data, RequestInfo.KnownDataKeys.BrowserMajorVersion) }, + { Alias.User, EventIndexExtensions.DataPath(Event.KnownDataKeys.UserInfo, u => u.Identity) }, + { Alias.UserName, EventIndexExtensions.DataPath(Event.KnownDataKeys.UserInfo, u => u.Name) }, + { Alias.UserEmail, EventIndexExtensions.DataPath(Event.KnownDataKeys.UserInfo, u => u.Identity) }, + { Alias.UserDescription, EventIndexExtensions.DataPath(Event.KnownDataKeys.UserDescription, u => u.Description) }, + { Alias.OperatingSystemVersion, EventIndexExtensions.DataDictionaryPath(Event.KnownDataKeys.RequestInfo, r => r.Data, RequestInfo.KnownDataKeys.OSVersion) }, + { Alias.OperatingSystemMajorVersion, EventIndexExtensions.DataDictionaryPath(Event.KnownDataKeys.RequestInfo, r => r.Data, RequestInfo.KnownDataKeys.OSMajorVersion) } }); } public ElasticsearchOptions Options => _configuration.Options; - private AnalysisDescriptor BuildAnalysis(AnalysisDescriptor ad) + private void BuildAnalysis(IndexSettingsAnalysisDescriptor ad) { - return ad.Analyzers(a => a + ad.Analyzers(a => a .Pattern(COMMA_WHITESPACE_ANALYZER, p => p.Pattern(@"[,\s]+")) - .Custom(EMAIL_ANALYZER, c => c.Filters(EMAIL_TOKEN_FILTER, "lowercase", TLD_STOPWORDS_TOKEN_FILTER, EDGE_NGRAM_TOKEN_FILTER, "unique").Tokenizer("keyword")) - .Custom(VERSION_INDEX_ANALYZER, c => c.Filters(VERSION_PAD1_TOKEN_FILTER, VERSION_PAD2_TOKEN_FILTER, VERSION_PAD3_TOKEN_FILTER, VERSION_PAD4_TOKEN_FILTER, VERSION_TOKEN_FILTER, "lowercase", "unique").Tokenizer("whitespace")) - .Custom(VERSION_SEARCH_ANALYZER, c => c.Filters(VERSION_PAD1_TOKEN_FILTER, VERSION_PAD2_TOKEN_FILTER, VERSION_PAD3_TOKEN_FILTER, VERSION_PAD4_TOKEN_FILTER, "lowercase").Tokenizer("whitespace")) - .Custom(WHITESPACE_LOWERCASE_ANALYZER, c => c.Filters("lowercase").Tokenizer(COMMA_WHITESPACE_TOKENIZER)) - .Custom(TYPENAME_ANALYZER, c => c.Filters(TYPENAME_TOKEN_FILTER, "lowercase", "unique").Tokenizer(TYPENAME_HIERARCHY_TOKENIZER)) - .Custom(STANDARDPLUS_ANALYZER, c => c.Filters(STANDARDPLUS_TOKEN_FILTER, "lowercase", "unique").Tokenizer(COMMA_WHITESPACE_TOKENIZER)) - .Custom(LOWER_KEYWORD_ANALYZER, c => c.Filters("lowercase").Tokenizer("keyword")) - .Custom(HOST_ANALYZER, c => c.Filters("lowercase").Tokenizer(HOST_TOKENIZER)) - .Custom(URL_PATH_ANALYZER, c => c.Filters("lowercase").Tokenizer(URL_PATH_TOKENIZER))) + .Custom(EMAIL_ANALYZER, c => c.Filter(EMAIL_TOKEN_FILTER, "lowercase", TLD_STOPWORDS_TOKEN_FILTER, EDGE_NGRAM_TOKEN_FILTER, "unique").Tokenizer("keyword")) + .Custom(VERSION_INDEX_ANALYZER, c => c.Filter(VERSION_PAD1_TOKEN_FILTER, VERSION_PAD2_TOKEN_FILTER, VERSION_PAD3_TOKEN_FILTER, VERSION_PAD4_TOKEN_FILTER, VERSION_TOKEN_FILTER, "lowercase", "unique").Tokenizer("whitespace")) + .Custom(VERSION_SEARCH_ANALYZER, c => c.Filter(VERSION_PAD1_TOKEN_FILTER, VERSION_PAD2_TOKEN_FILTER, VERSION_PAD3_TOKEN_FILTER, VERSION_PAD4_TOKEN_FILTER, "lowercase").Tokenizer("whitespace")) + .Custom(WHITESPACE_LOWERCASE_ANALYZER, c => c.Filter("lowercase").Tokenizer(COMMA_WHITESPACE_TOKENIZER)) + .Custom(TYPENAME_ANALYZER, c => c.Filter(TYPENAME_TOKEN_FILTER, "lowercase", "unique").Tokenizer(TYPENAME_HIERARCHY_TOKENIZER)) + .Custom(STANDARDPLUS_ANALYZER, c => c.Filter(STANDARDPLUS_TOKEN_FILTER, "lowercase", "unique").Tokenizer(COMMA_WHITESPACE_TOKENIZER)) + .Custom(LOWER_KEYWORD_ANALYZER, c => c.Filter("lowercase").Tokenizer("keyword")) + .Custom(HOST_ANALYZER, c => c.Filter("lowercase").Tokenizer(HOST_TOKENIZER)) + .Custom(URL_PATH_ANALYZER, c => c.Filter("lowercase").Tokenizer(URL_PATH_TOKENIZER))) .TokenFilters(f => f - .EdgeNGram(EDGE_NGRAM_TOKEN_FILTER, p => p.MaxGram(50).MinGram(2).Side(EdgeNGramSide.Front)) - .PatternCapture(EMAIL_TOKEN_FILTER, p => p.PreserveOriginal().Patterns("(\\w+)", "(\\p{L}+)", "(\\d+)", "@(.+)", "@(.+)\\.", "(.+)@")) - .PatternCapture(STANDARDPLUS_TOKEN_FILTER, p => p.PreserveOriginal().Patterns( + .EdgeNGram(EDGE_NGRAM_TOKEN_FILTER, p => p.MaxGram(50).MinGram(2).Side(Elastic.Clients.Elasticsearch.Analysis.EdgeNGramSide.Front)) + .PatternCapture(EMAIL_TOKEN_FILTER, p => p.PreserveOriginal(true).Patterns("(\\w+)", "(\\p{L}+)", "(\\d+)", "@(.+)", "@(.+)\\.", "(.+)@")) + .PatternCapture(STANDARDPLUS_TOKEN_FILTER, p => p.PreserveOriginal(true).Patterns( @"([^\.\(\)\[\]\/\\\{\}\?=&;:\<\>]+)", @"([^\.\(\)\[\]\/\\\{\}\?=&;:\<\>]+[\.\/\\][^\.\(\)\[\]\/\\\{\}\?=&;:\<\>]+)", @"([^\.\(\)\[\]\/\\\{\}\?=&;:\<\>]+[\.\/\\][^\.\(\)\[\]\/\\\{\}\?=&;:\<\>]+[\.\/\\][^\.\(\)\[\]\/\\\{\}\?=&;:\<\>]+)" )) - .PatternCapture(TYPENAME_TOKEN_FILTER, p => p.PreserveOriginal().Patterns(@" ^ (\w+)", @"\.(\w+)", @"([^\(\)]+)")) + .PatternCapture(TYPENAME_TOKEN_FILTER, p => p.PreserveOriginal(true).Patterns(@" ^ (\w+)", @"\.(\w+)", @"([^\(\)]+)")) .PatternCapture(VERSION_TOKEN_FILTER, p => p.Patterns(@"^(\d+)\.", @"^(\d+\.\d+)", @"^(\d+\.\d+\.\d+)")) .PatternReplace(VERSION_PAD1_TOKEN_FILTER, p => p.Pattern(@"(\.|^)(\d{1})(?=\.|-|$)").Replacement("$10000$2")) .PatternReplace(VERSION_PAD2_TOKEN_FILTER, p => p.Pattern(@"(\.|^)(\d{2})(?=\.|-|$)").Replacement("$1000$2")) .PatternReplace(VERSION_PAD3_TOKEN_FILTER, p => p.Pattern(@"(\.|^)(\d{3})(?=\.|-|$)").Replacement("$100$2")) .PatternReplace(VERSION_PAD4_TOKEN_FILTER, p => p.Pattern(@"(\.|^)(\d{4})(?=\.|-|$)").Replacement("$10$2")) - .Stop(TLD_STOPWORDS_TOKEN_FILTER, p => p.StopWords("com", "net", "org", "info", "me", "edu", "mil", "gov", "biz", "co", "io", "dev")) - .WordDelimiter(ALL_WORDS_DELIMITER_TOKEN_FILTER, p => p.CatenateNumbers().PreserveOriginal().CatenateAll().CatenateWords())) + .Stop(TLD_STOPWORDS_TOKEN_FILTER, p => p.Stopwords(new string[] { "com", "net", "org", "info", "me", "edu", "mil", "gov", "biz", "co", "io", "dev" })) + .WordDelimiter(ALL_WORDS_DELIMITER_TOKEN_FILTER, p => p.CatenateNumbers(true).PreserveOriginal(true).CatenateAll(true).CatenateWords(true))) .Tokenizers(t => t - .CharGroup(COMMA_WHITESPACE_TOKENIZER, p => p.TokenizeOnCharacters(",", "whitespace")) - .CharGroup(URL_PATH_TOKENIZER, p => p.TokenizeOnCharacters("/", "-", ".")) - .CharGroup(HOST_TOKENIZER, p => p.TokenizeOnCharacters(".")) - .PathHierarchy(TYPENAME_HIERARCHY_TOKENIZER, p => p.Delimiter('.'))); + .CharGroup(COMMA_WHITESPACE_TOKENIZER, p => p.TokenizeOnChars(",", "whitespace")) + .CharGroup(URL_PATH_TOKENIZER, p => p.TokenizeOnChars("/", "-", ".")) + .CharGroup(HOST_TOKENIZER, p => p.TokenizeOnChars(".")) + .PathHierarchy(TYPENAME_HIERARCHY_TOKENIZER, p => p.Delimiter("."))); } private const string ALL_WORDS_DELIMITER_TOKEN_FILTER = "all_word_delimiter"; @@ -318,131 +322,169 @@ public sealed class Alias internal static class EventIndexExtensions { + private static readonly JsonSerializerOptions _propertyNameOptions = new JsonSerializerOptions().ConfigureExceptionlessDefaults(); + public static PropertiesDescriptor AddCopyToMappings(this PropertiesDescriptor descriptor) { return descriptor - .Text(f => f.Name(EventIndex.Alias.IpAddress).Analyzer(EventIndex.COMMA_WHITESPACE_ANALYZER).AddKeywordField()) - .Text(f => f.Name(EventIndex.Alias.OperatingSystem).Analyzer(EventIndex.WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField()) - .Object(f => f.Name(EventIndex.Alias.Error).Properties(p1 => p1 - .Keyword(f3 => f3.Name("code").IgnoreAbove(1024)) - .Text(f3 => f3.Name("message").AddKeywordField()) - .Text(f3 => f3.Name("type").Analyzer(EventIndex.TYPENAME_ANALYZER).SearchAnalyzer(EventIndex.WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField()) - .Text(f6 => f6.Name("targettype").Analyzer(EventIndex.TYPENAME_ANALYZER).SearchAnalyzer(EventIndex.WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField()) - .Text(f6 => f6.Name("targetmethod").Analyzer(EventIndex.TYPENAME_ANALYZER).SearchAnalyzer(EventIndex.WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField()))); + .Text(EventIndex.Alias.IpAddress, t => t.Analyzer(EventIndex.COMMA_WHITESPACE_ANALYZER).AddKeywordField()) + .Text(EventIndex.Alias.OperatingSystem, t => t.Analyzer(EventIndex.WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField()) + .Object(EventIndex.Alias.Error, o => o.Properties(p1 => p1 + .Keyword("code", k => k.IgnoreAbove(1024)) + .Text("message", t => t.AddKeywordField()) + .Text("type", t => t.Analyzer(EventIndex.TYPENAME_ANALYZER).SearchAnalyzer(EventIndex.WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField()) + .Text("targettype", t => t.Analyzer(EventIndex.TYPENAME_ANALYZER).SearchAnalyzer(EventIndex.WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField()) + .Text("targetmethod", t => t.Analyzer(EventIndex.TYPENAME_ANALYZER).SearchAnalyzer(EventIndex.WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField()))); } public static PropertiesDescriptor AddDataDictionaryAliases(this PropertiesDescriptor descriptor) { return descriptor - .FieldAlias(a => a.Name(EventIndex.Alias.Version).Path(f => (string)f.Data![Event.KnownDataKeys.Version]!)) - .FieldAlias(a => a.Name(EventIndex.Alias.Level).Path(f => (string)f.Data![Event.KnownDataKeys.Level]!)) - .FieldAlias(a => a.Name(EventIndex.Alias.SubmissionMethod).Path(f => (string)f.Data![Event.KnownDataKeys.SubmissionMethod]!)) - .FieldAlias(a => a.Name(EventIndex.Alias.ClientUserAgent).Path(f => ((SubmissionClient)f.Data![Event.KnownDataKeys.SubmissionClient]!).UserAgent)) - .FieldAlias(a => a.Name(EventIndex.Alias.ClientVersion).Path(f => ((SubmissionClient)f.Data![Event.KnownDataKeys.SubmissionClient]!).Version)) - .FieldAlias(a => a.Name(EventIndex.Alias.LocationCountry).Path(f => ((Location)f.Data![Event.KnownDataKeys.Location]!).Country)) - .FieldAlias(a => a.Name(EventIndex.Alias.LocationLevel1).Path(f => ((Location)f.Data![Event.KnownDataKeys.Location]!).Level1)) - .FieldAlias(a => a.Name(EventIndex.Alias.LocationLevel2).Path(f => ((Location)f.Data![Event.KnownDataKeys.Location]!).Level2)) - .FieldAlias(a => a.Name(EventIndex.Alias.LocationLocality).Path(f => ((Location)f.Data![Event.KnownDataKeys.Location]!).Locality)) - .FieldAlias(a => a.Name(EventIndex.Alias.Browser).Path(f => ((RequestInfo)f.Data![Event.KnownDataKeys.RequestInfo]!).Data![RequestInfo.KnownDataKeys.Browser])) - .FieldAlias(a => a.Name(EventIndex.Alias.Device).Path(f => ((RequestInfo)f.Data![Event.KnownDataKeys.RequestInfo]!).Data![RequestInfo.KnownDataKeys.Device])) - .FieldAlias(a => a.Name(EventIndex.Alias.RequestIsBot).Path(f => ((RequestInfo)f.Data![Event.KnownDataKeys.RequestInfo]!).Data![RequestInfo.KnownDataKeys.IsBot])) - .FieldAlias(a => a.Name(EventIndex.Alias.RequestPath).Path(f => ((RequestInfo)f.Data![Event.KnownDataKeys.RequestInfo]!).Path)) - .FieldAlias(a => a.Name(EventIndex.Alias.RequestUserAgent).Path(f => ((RequestInfo)f.Data![Event.KnownDataKeys.RequestInfo]!).UserAgent)) - .FieldAlias(a => a.Name(EventIndex.Alias.CommandLine).Path(f => ((EnvironmentInfo)f.Data![Event.KnownDataKeys.EnvironmentInfo]!).CommandLine)) - .FieldAlias(a => a.Name(EventIndex.Alias.MachineArchitecture).Path(f => ((EnvironmentInfo)f.Data![Event.KnownDataKeys.EnvironmentInfo]!).Architecture)) - .FieldAlias(a => a.Name(EventIndex.Alias.MachineName).Path(f => ((EnvironmentInfo)f.Data![Event.KnownDataKeys.EnvironmentInfo]!).MachineName)); + .FieldAlias(EventIndex.Alias.Version, a => a.Path($"data.{Event.KnownDataKeys.Version}")) + .FieldAlias(EventIndex.Alias.Level, a => a.Path($"data.{Event.KnownDataKeys.Level}")) + .FieldAlias(EventIndex.Alias.SubmissionMethod, a => a.Path($"data.{Event.KnownDataKeys.SubmissionMethod}")) + .FieldAlias(EventIndex.Alias.ClientUserAgent, a => a.Path(DataPath(Event.KnownDataKeys.SubmissionClient, c => c.UserAgent))) + .FieldAlias(EventIndex.Alias.ClientVersion, a => a.Path(DataPath(Event.KnownDataKeys.SubmissionClient, c => c.Version))) + .FieldAlias(EventIndex.Alias.LocationCountry, a => a.Path(DataPath(Event.KnownDataKeys.Location, l => l.Country))) + .FieldAlias(EventIndex.Alias.LocationLevel1, a => a.Path(DataPath(Event.KnownDataKeys.Location, l => l.Level1))) + .FieldAlias(EventIndex.Alias.LocationLevel2, a => a.Path(DataPath(Event.KnownDataKeys.Location, l => l.Level2))) + .FieldAlias(EventIndex.Alias.LocationLocality, a => a.Path(DataPath(Event.KnownDataKeys.Location, l => l.Locality))) + .FieldAlias(EventIndex.Alias.Browser, a => a.Path(DataDictionaryPath(Event.KnownDataKeys.RequestInfo, r => r.Data, RequestInfo.KnownDataKeys.Browser))) + .FieldAlias(EventIndex.Alias.Device, a => a.Path(DataDictionaryPath(Event.KnownDataKeys.RequestInfo, r => r.Data, RequestInfo.KnownDataKeys.Device))) + .FieldAlias(EventIndex.Alias.RequestIsBot, a => a.Path(DataDictionaryPath(Event.KnownDataKeys.RequestInfo, r => r.Data, RequestInfo.KnownDataKeys.IsBot))) + .FieldAlias(EventIndex.Alias.RequestPath, a => a.Path(DataPath(Event.KnownDataKeys.RequestInfo, r => r.Path))) + .FieldAlias(EventIndex.Alias.RequestUserAgent, a => a.Path(DataPath(Event.KnownDataKeys.RequestInfo, r => r.UserAgent))) + .FieldAlias(EventIndex.Alias.CommandLine, a => a.Path(DataPath(Event.KnownDataKeys.EnvironmentInfo, e => e.CommandLine))) + .FieldAlias(EventIndex.Alias.MachineArchitecture, a => a.Path(DataPath(Event.KnownDataKeys.EnvironmentInfo, e => e.Architecture))) + .FieldAlias(EventIndex.Alias.MachineName, a => a.Path(DataPath(Event.KnownDataKeys.EnvironmentInfo, e => e.MachineName))); + } + + public static PropertiesDescriptor AddVersionMapping(this PropertiesDescriptor descriptor) where T : class + { + return descriptor.Text(Event.KnownDataKeys.Version, t => t.Analyzer(EventIndex.VERSION_INDEX_ANALYZER).SearchAnalyzer(EventIndex.VERSION_SEARCH_ANALYZER).AddKeywordField()); + } + + public static PropertiesDescriptor AddLevelMapping(this PropertiesDescriptor descriptor) where T : class + { + return descriptor.Text(Event.KnownDataKeys.Level, t => t.Analyzer(EventIndex.LOWER_KEYWORD_ANALYZER).AddKeywordField()); + } + + public static PropertiesDescriptor AddSubmissionMethodMapping(this PropertiesDescriptor descriptor) where T : class + { + return descriptor.Keyword(Event.KnownDataKeys.SubmissionMethod, k => k.IgnoreAbove(1024)); } - public static PropertiesDescriptor AddVersionMapping(this PropertiesDescriptor descriptor) + public static PropertiesDescriptor AddSubmissionClientMapping(this PropertiesDescriptor descriptor) where T : class { - return descriptor.Text(f2 => f2.Name(Event.KnownDataKeys.Version).Analyzer(EventIndex.VERSION_INDEX_ANALYZER).SearchAnalyzer(EventIndex.VERSION_SEARCH_ANALYZER).AddKeywordField()); + return descriptor.Object(Event.KnownDataKeys.SubmissionClient, o => o.Properties(p3 => p3 + .Text(Field(c => c.IpAddress), t => t.Analyzer(EventIndex.COMMA_WHITESPACE_ANALYZER).CopyTo(EventIndex.Alias.IpAddress)) + .Text(Field(c => c.UserAgent), t => t.Analyzer(EventIndex.LOWER_KEYWORD_ANALYZER).AddKeywordField()) + .Keyword(Field(c => c.Version), k => k.IgnoreAbove(1024)))); } - public static PropertiesDescriptor AddLevelMapping(this PropertiesDescriptor descriptor) + public static PropertiesDescriptor AddLocationMapping(this PropertiesDescriptor descriptor) where T : class { - return descriptor.Text(f2 => f2.Name(Event.KnownDataKeys.Level).Analyzer(EventIndex.LOWER_KEYWORD_ANALYZER).AddKeywordField()); + return descriptor.Object(Event.KnownDataKeys.Location, o => o.Properties(p3 => p3 + .Text(Field(l => l.Country), t => t.Analyzer(EventIndex.LOWER_KEYWORD_ANALYZER).AddKeywordField()) + .Keyword(Field(l => l.Level1), k => k.IgnoreAbove(1024)) + .Keyword(Field(l => l.Level2), k => k.IgnoreAbove(1024)) + .Keyword(Field(l => l.Locality), k => k.IgnoreAbove(1024)))); } - public static PropertiesDescriptor AddSubmissionMethodMapping(this PropertiesDescriptor descriptor) + public static PropertiesDescriptor AddRequestInfoMapping(this PropertiesDescriptor descriptor) where T : class { - return descriptor.Keyword(f2 => f2.Name(Event.KnownDataKeys.SubmissionMethod).IgnoreAbove(1024)); + return descriptor.Object(Event.KnownDataKeys.RequestInfo, o => o.Properties(p3 => p3 + .Text(Field(r => r.ClientIpAddress), t => t.Analyzer(EventIndex.COMMA_WHITESPACE_ANALYZER).CopyTo(EventIndex.Alias.IpAddress)) + .Text(Field(r => r.UserAgent), t => t.AddKeywordField()) + .Text(Field(r => r.Path), t => t.Analyzer(EventIndex.URL_PATH_ANALYZER).AddKeywordField()) + .Text(Field(r => r.Host), t => t.Analyzer(EventIndex.HOST_ANALYZER).AddKeywordField()) + .IntegerNumber(Field(r => r.Port)) + .Keyword(Field(r => r.HttpMethod)) + .Object(Field(r => r.Data), oi => oi.Properties(p4 => p4 + .Text(RequestInfo.KnownDataKeys.Browser, t => t.Analyzer(EventIndex.WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField()) + .Keyword(RequestInfo.KnownDataKeys.BrowserVersion, k => k.IgnoreAbove(1024)) + .Keyword(RequestInfo.KnownDataKeys.BrowserMajorVersion, k => k.IgnoreAbove(1024)) + .Text(RequestInfo.KnownDataKeys.Device, t => t.Analyzer(EventIndex.WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField()) + .Text(RequestInfo.KnownDataKeys.OS, t => t.Analyzer(EventIndex.WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField().CopyTo(EventIndex.Alias.OperatingSystem).Index(false)) + .Keyword(RequestInfo.KnownDataKeys.OSVersion, k => k.IgnoreAbove(1024)) + .Keyword(RequestInfo.KnownDataKeys.OSMajorVersion, k => k.IgnoreAbove(1024)) + .Boolean(RequestInfo.KnownDataKeys.IsBot))))); } - public static PropertiesDescriptor AddSubmissionClientMapping(this PropertiesDescriptor descriptor) + public static PropertiesDescriptor AddErrorMapping(this PropertiesDescriptor descriptor) where T : class { - return descriptor.Object(f2 => f2.Name(Event.KnownDataKeys.SubmissionClient).Properties(p3 => p3 - .Text(f3 => f3.Name(r => r.IpAddress).Analyzer(EventIndex.COMMA_WHITESPACE_ANALYZER).CopyTo(fd => fd.Field(EventIndex.Alias.IpAddress))) - .Text(f3 => f3.Name(r => r.UserAgent).Analyzer(EventIndex.LOWER_KEYWORD_ANALYZER).AddKeywordField()) - .Keyword(f3 => f3.Name(r => r.Version).IgnoreAbove(1024)))); + return descriptor.Object(Event.KnownDataKeys.Error, o => o.Properties(p3 => p3 + .Object(Field(e => e.Data), oi => oi.Properties(p4 => p4 + .Object(Error.KnownDataKeys.TargetInfo, oi2 => oi2.Properties(p5 => p5 + .Keyword("ExceptionType", k => k.IgnoreAbove(1024).CopyTo(EventIndex.Alias.ErrorTargetType)) + .Keyword("Method", k => k.IgnoreAbove(1024).CopyTo(EventIndex.Alias.ErrorTargetMethod)))))))); } - public static PropertiesDescriptor AddLocationMapping(this PropertiesDescriptor descriptor) + public static PropertiesDescriptor AddSimpleErrorMapping(this PropertiesDescriptor descriptor) where T : class { - return descriptor.Object(f2 => f2.Name(Event.KnownDataKeys.Location).Properties(p3 => p3 - .Text(f3 => f3.Name(r => r.Country).Analyzer(EventIndex.LOWER_KEYWORD_ANALYZER).AddKeywordField()) - .Keyword(f3 => f3.Name(r => r.Level1).IgnoreAbove(1024)) - .Keyword(f3 => f3.Name(r => r.Level2).IgnoreAbove(1024)) - .Keyword(f3 => f3.Name(r => r.Locality).IgnoreAbove(1024)))); + return descriptor.Object(Event.KnownDataKeys.SimpleError, o => o.Properties(p3 => p3 + .Object(Field(e => e.Data), oi => oi.Properties(p4 => p4 + .Object(Error.KnownDataKeys.TargetInfo, oi2 => oi2.Properties(p5 => p5 + .Keyword("ExceptionType", k => k.IgnoreAbove(1024).CopyTo(EventIndex.Alias.ErrorTargetType)))))))); } - public static PropertiesDescriptor AddRequestInfoMapping(this PropertiesDescriptor descriptor) + public static PropertiesDescriptor AddEnvironmentInfoMapping(this PropertiesDescriptor descriptor) where T : class { - return descriptor.Object(f2 => f2.Name(Event.KnownDataKeys.RequestInfo).Properties(p3 => p3 - .Text(f3 => f3.Name(r => r.ClientIpAddress).Analyzer(EventIndex.COMMA_WHITESPACE_ANALYZER).CopyTo(fd => fd.Field(EventIndex.Alias.IpAddress))) - .Text(f3 => f3.Name(r => r.UserAgent).AddKeywordField()) - .Text(f3 => f3.Name(r => r.Path).Analyzer(EventIndex.URL_PATH_ANALYZER).AddKeywordField()) - .Text(f3 => f3.Name(r => r.Host).Analyzer(EventIndex.HOST_ANALYZER).AddKeywordField()) - .Scalar(r => r.Port) - .Keyword(f3 => f3.Name(r => r.HttpMethod)) - .Object(f3 => f3.Name(e => e.Data).Properties(p4 => p4 - .Text(f4 => f4.Name(RequestInfo.KnownDataKeys.Browser).Analyzer(EventIndex.WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField()) - .Keyword(f4 => f4.Name(RequestInfo.KnownDataKeys.BrowserVersion).IgnoreAbove(1024)) - .Keyword(f4 => f4.Name(RequestInfo.KnownDataKeys.BrowserMajorVersion).IgnoreAbove(1024)) - .Text(f4 => f4.Name(RequestInfo.KnownDataKeys.Device).Analyzer(EventIndex.WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField()) - .Text(f4 => f4.Name(RequestInfo.KnownDataKeys.OS).Analyzer(EventIndex.WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField().CopyTo(fd => fd.Field(EventIndex.Alias.OperatingSystem)).Index(false)) - .Keyword(f4 => f4.Name(RequestInfo.KnownDataKeys.OSVersion).IgnoreAbove(1024)) - .Keyword(f4 => f4.Name(RequestInfo.KnownDataKeys.OSMajorVersion).IgnoreAbove(1024)) - .Boolean(f4 => f4.Name(RequestInfo.KnownDataKeys.IsBot)))))); + return descriptor.Object(Event.KnownDataKeys.EnvironmentInfo, o => o.Properties(p3 => p3 + .Text(Field(e => e.IpAddress), t => t.Analyzer(EventIndex.COMMA_WHITESPACE_ANALYZER).CopyTo(EventIndex.Alias.IpAddress)) + .Text(Field(e => e.MachineName), t => t.Analyzer(EventIndex.LOWER_KEYWORD_ANALYZER).AddKeywordField()) + .Text(Field(e => e.OSName), t => t.Analyzer(EventIndex.WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField().CopyTo(EventIndex.Alias.OperatingSystem)) + .Keyword(Field(e => e.CommandLine), k => k.IgnoreAbove(1024)) + .Keyword(Field(e => e.Architecture), k => k.IgnoreAbove(1024)))); } - public static PropertiesDescriptor AddErrorMapping(this PropertiesDescriptor descriptor) + public static PropertiesDescriptor AddUserDescriptionMapping(this PropertiesDescriptor descriptor) where T : class { - return descriptor.Object(f2 => f2.Name(Event.KnownDataKeys.Error).Properties(p3 => p3 - .Object(f4 => f4.Name(e => e.Data).Properties(p4 => p4 - .Object(f5 => f5.Name(Error.KnownDataKeys.TargetInfo).Properties(p5 => p5 - .Keyword(f6 => f6.Name("ExceptionType").IgnoreAbove(1024).CopyTo(fd => fd.Field(EventIndex.Alias.ErrorTargetType))) - .Keyword(f6 => f6.Name("Method").IgnoreAbove(1024).CopyTo(fd => fd.Field(EventIndex.Alias.ErrorTargetMethod))))))))); + return descriptor.Object(Event.KnownDataKeys.UserDescription, o => o.Properties(p3 => p3 + .Text(Field(u => u.Description)) + .Text(Field(u => u.EmailAddress), t => t.Analyzer(EventIndex.EMAIL_ANALYZER).SearchAnalyzer("simple").AddKeywordField().CopyTo(DataPath(Event.KnownDataKeys.UserInfo, u => u.Identity))))); } - public static PropertiesDescriptor AddSimpleErrorMapping(this PropertiesDescriptor descriptor) + public static PropertiesDescriptor AddUserInfoMapping(this PropertiesDescriptor descriptor) where T : class { - return descriptor.Object(f2 => f2.Name(Event.KnownDataKeys.SimpleError).Properties(p3 => p3 - .Object(f4 => f4.Name(e => e.Data).Properties(p4 => p4 - .Object(f5 => f5.Name(Error.KnownDataKeys.TargetInfo).Properties(p5 => p5 - .Keyword(f6 => f6.Name("ExceptionType").IgnoreAbove(1024).CopyTo(fd => fd.Field(EventIndex.Alias.ErrorTargetType))))))))); + return descriptor.Object(Event.KnownDataKeys.UserInfo, o => o.Properties(p3 => p3 + .Text(Field(u => u.Identity), t => t.Analyzer(EventIndex.EMAIL_ANALYZER).SearchAnalyzer(EventIndex.WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField()) + .Text(Field(u => u.Name), t => t.Analyzer(EventIndex.LOWER_KEYWORD_ANALYZER).AddKeywordField()))); } - public static PropertiesDescriptor AddEnvironmentInfoMapping(this PropertiesDescriptor descriptor) + public static string DataPath(string dataKey, Expression> property) { - return descriptor.Object(f2 => f2.Name(Event.KnownDataKeys.EnvironmentInfo).Properties(p3 => p3 - .Text(f3 => f3.Name(r => r.IpAddress).Analyzer(EventIndex.COMMA_WHITESPACE_ANALYZER).CopyTo(fd => fd.Field(EventIndex.Alias.IpAddress))) - .Text(f3 => f3.Name(r => r.MachineName).Analyzer(EventIndex.LOWER_KEYWORD_ANALYZER).AddKeywordField()) - .Text(f3 => f3.Name(r => r.OSName).Analyzer(EventIndex.WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField().CopyTo(fd => fd.Field(EventIndex.Alias.OperatingSystem))) - .Keyword(f3 => f3.Name(r => r.CommandLine).IgnoreAbove(1024)) - .Keyword(f3 => f3.Name(r => r.Architecture).IgnoreAbove(1024)))); + return $"data.{dataKey}.{Field(property)}"; } - public static PropertiesDescriptor AddUserDescriptionMapping(this PropertiesDescriptor descriptor) + public static string DataDictionaryPath(string dataKey, Expression> dictionaryProperty, string dictionaryKey) { - return descriptor.Object(f2 => f2.Name(Event.KnownDataKeys.UserDescription).Properties(p3 => p3 - .Text(f3 => f3.Name(r => r.Description)) - .Text(f3 => f3.Name(r => r.EmailAddress).Analyzer(EventIndex.EMAIL_ANALYZER).SearchAnalyzer("simple").AddKeywordField().CopyTo(f4 => f4.Field($"data.{Event.KnownDataKeys.UserInfo}.identity"))))); + return $"{DataPath(dataKey, dictionaryProperty)}.{dictionaryKey}"; + } + + private static string Field(Expression> property) + { + string propertyName = GetPropertyInfo(property).Name; + JsonTypeInfo typeInfo = _propertyNameOptions.GetTypeInfo(typeof(TModel)); + + foreach (JsonPropertyInfo jsonProperty in typeInfo.Properties) + { + if (jsonProperty.AttributeProvider is PropertyInfo modelProperty && String.Equals(modelProperty.Name, propertyName, StringComparison.Ordinal)) + return jsonProperty.Name; + } + + throw new InvalidOperationException($"Unable to resolve JSON field name for {typeof(TModel).FullName}.{propertyName}."); } - public static PropertiesDescriptor AddUserInfoMapping(this PropertiesDescriptor descriptor) + private static PropertyInfo GetPropertyInfo(Expression> expression) { - return descriptor.Object(f2 => f2.Name(Event.KnownDataKeys.UserInfo).Properties(p3 => p3 - .Text(f3 => f3.Name(r => r.Identity).Analyzer(EventIndex.EMAIL_ANALYZER).SearchAnalyzer(EventIndex.WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField()) - .Text(f3 => f3.Name(r => r.Name).Analyzer(EventIndex.LOWER_KEYWORD_ANALYZER).AddKeywordField()))); + Expression body = expression.Body is UnaryExpression { NodeType: ExpressionType.Convert } unary + ? unary.Operand + : expression.Body; + + if (body is MemberExpression { Member: PropertyInfo property }) + return property; + + throw new ArgumentException("Expression must select a model property.", nameof(expression)); } } diff --git a/src/Exceptionless.Core/Repositories/Configuration/Indexes/OrganizationIndex.cs b/src/Exceptionless.Core/Repositories/Configuration/Indexes/OrganizationIndex.cs index 11a487b319..4550ab40f7 100644 --- a/src/Exceptionless.Core/Repositories/Configuration/Indexes/OrganizationIndex.cs +++ b/src/Exceptionless.Core/Repositories/Configuration/Indexes/OrganizationIndex.cs @@ -1,7 +1,9 @@ -using Exceptionless.Core.Models; +using Elastic.Clients.Elasticsearch.IndexManagement; +using Elastic.Clients.Elasticsearch.Mapping; +using Exceptionless.Core.Models; +using Foundatio.Parsers.ElasticQueries.Extensions; using Foundatio.Repositories.Elasticsearch.Configuration; using Foundatio.Repositories.Elasticsearch.Extensions; -using Nest; namespace Exceptionless.Core.Repositories.Configuration; @@ -15,37 +17,38 @@ public OrganizationIndex(ExceptionlessElasticConfiguration configuration) : base _configuration = configuration; } - public override TypeMappingDescriptor ConfigureIndexMapping(TypeMappingDescriptor map) + public override void ConfigureIndexMapping(TypeMappingDescriptor map) { - return map - .Dynamic(false) + map + .Dynamic(DynamicMapping.False) .Properties(p => p .SetupDefaults() - .Text(f => f.Name(e => e.Name).AddKeywordField()) - .Keyword(f => f.Name(u => u.StripeCustomerId)) - .Boolean(f => f.Name(u => u.HasPremiumFeatures)) - .Keyword(f => f.Name(u => u.Features)) - .Keyword(f => f.Name(u => u.PlanId)) - .Keyword(f => f.Name(u => u.PlanName).IgnoreAbove(256)) - .Date(f => f.Name(u => u.SubscribeDate)) - .Number(f => f.Name(u => u.BillingStatus)) - .Scalar(f => f.BillingPrice, f => f) - .Boolean(f => f.Name(u => u.IsSuspended)) - .Scalar(f => f.RetentionDays, f => f) - .Object(f => f.Name(o => o.Invites.First()).Properties(ip => ip - .Keyword(fu => fu.Name(i => i.Token)) - .Text(fu => fu.Name(i => i.EmailAddress).Analyzer(KEYWORD_LOWERCASE_ANALYZER)))) - .Date(f => f.Name(s => s.LastEventDateUtc)) + .Text(e => e.Name, t => t.AddKeywordField()) + .Keyword(e => e.StripeCustomerId) + .Boolean(e => e.HasPremiumFeatures) + .Keyword(e => e.Features) + .Keyword(e => e.PlanId) + .Keyword(e => e.PlanName, k => k.IgnoreAbove(256)) + .Date(e => e.SubscribeDate) + .FloatNumber(e => e.BillingStatus) + .DoubleNumber(e => e.BillingPrice) + .Boolean(e => e.IsSuspended) + .IntegerNumber(e => e.RetentionDays) + .Object(e => e.Invites, o => o.Properties(ip => ip + .Keyword("token") + .Text("email_address", t => t.Analyzer(KEYWORD_LOWERCASE_ANALYZER)))) + .Date(e => e.LastEventDateUtc) .AddUsageMappings()); } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - return base.ConfigureIndex(idx.Settings(s => s - .Analysis(d => d.Analyzers(b => b.Custom(KEYWORD_LOWERCASE_ANALYZER, c => c.Filters("lowercase").Tokenizer("keyword")))) + base.ConfigureIndex(idx); + idx.Settings(s => s + .Analysis(d => d.Analyzers(b => b.Custom(KEYWORD_LOWERCASE_ANALYZER, c => c.Filter("lowercase").Tokenizer("keyword")))) .NumberOfShards(_configuration.Options.NumberOfShards) .NumberOfReplicas(_configuration.Options.NumberOfReplicas) - .Priority(10))); + .Priority(10)); } } @@ -54,19 +57,19 @@ internal static class OrganizationIndexExtensions public static PropertiesDescriptor AddUsageMappings(this PropertiesDescriptor descriptor) { return descriptor - .Object(ui => ui.Name(o => o.Usage.First()).Properties(p => p - .Date(fu => fu.Name(i => i.Date)) - .Number(fu => fu.Name(i => i.Total)) - .Number(fu => fu.Name(i => i.Blocked)) - .Number(fu => fu.Name(i => i.Discarded)) - .Number(fu => fu.Name(i => i.Limit)) - .Number(fu => fu.Name(i => i.TooBig)))) - .Object(ui => ui.Name(o => o.UsageHours.First()).Properties(p => p - .Date(fu => fu.Name(i => i.Date)) - .Number(fu => fu.Name(i => i.Total)) - .Number(fu => fu.Name(i => i.Blocked)) - .Number(fu => fu.Name(i => i.Discarded)) - .Number(fu => fu.Name(i => i.Limit)) - .Number(fu => fu.Name(i => i.TooBig)))); + .Object(o => o.Usage, ui => ui.Properties(p => p + .Date("date") + .FloatNumber("total") + .FloatNumber("blocked") + .FloatNumber("discarded") + .FloatNumber("limit") + .FloatNumber("too_big"))) + .Object(o => o.UsageHours, ui => ui.Properties(p => p + .Date("date") + .FloatNumber("total") + .FloatNumber("blocked") + .FloatNumber("discarded") + .FloatNumber("limit") + .FloatNumber("too_big"))); } } diff --git a/src/Exceptionless.Core/Repositories/Configuration/Indexes/ProjectIndex.cs b/src/Exceptionless.Core/Repositories/Configuration/Indexes/ProjectIndex.cs index 38585e9061..c3eec0b34f 100644 --- a/src/Exceptionless.Core/Repositories/Configuration/Indexes/ProjectIndex.cs +++ b/src/Exceptionless.Core/Repositories/Configuration/Indexes/ProjectIndex.cs @@ -1,7 +1,9 @@ -using Exceptionless.Core.Models; +using Elastic.Clients.Elasticsearch.IndexManagement; +using Elastic.Clients.Elasticsearch.Mapping; +using Exceptionless.Core.Models; +using Foundatio.Parsers.ElasticQueries.Extensions; using Foundatio.Repositories.Elasticsearch.Configuration; using Foundatio.Repositories.Elasticsearch.Extensions; -using Nest; namespace Exceptionless.Core.Repositories.Configuration; @@ -15,28 +17,28 @@ public ProjectIndex(ExceptionlessElasticConfiguration configuration) : base(conf _configuration = configuration; } - - public override TypeMappingDescriptor ConfigureIndexMapping(TypeMappingDescriptor map) + public override void ConfigureIndexMapping(TypeMappingDescriptor map) { - return map - .Dynamic(false) + map + .Dynamic(DynamicMapping.False) .Properties(p => p .SetupDefaults() - .Keyword(f => f.Name(e => e.OrganizationId)) - .Text(f => f.Name(e => e.Name).AddKeywordField()) - .Scalar(f => f.NextSummaryEndOfDayTicks, f => f) - .Date(f => f.Name(s => s.LastEventDateUtc)) + .Keyword(e => e.OrganizationId) + .Text(e => e.Name, t => t.AddKeywordField()) + .LongNumber(e => e.NextSummaryEndOfDayTicks) + .Date(e => e.LastEventDateUtc) .AddUsageMappings() ); } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - return base.ConfigureIndex(idx.Settings(s => s - .Analysis(d => d.Analyzers(b => b.Custom(KEYWORD_LOWERCASE_ANALYZER, c => c.Filters("lowercase").Tokenizer("keyword")))) + base.ConfigureIndex(idx); + idx.Settings(s => s + .Analysis(d => d.Analyzers(b => b.Custom(KEYWORD_LOWERCASE_ANALYZER, c => c.Filter("lowercase").Tokenizer("keyword")))) .NumberOfShards(_configuration.Options.NumberOfShards) .NumberOfReplicas(_configuration.Options.NumberOfReplicas) - .Priority(10))); + .Priority(10)); } } @@ -45,19 +47,19 @@ internal static class ProjectIndexExtensions public static PropertiesDescriptor AddUsageMappings(this PropertiesDescriptor descriptor) { return descriptor - .Object(ui => ui.Name(o => o.Usage.First()).Properties(p => p - .Date(fu => fu.Name(i => i.Date)) - .Number(fu => fu.Name(i => i.Total)) - .Number(fu => fu.Name(i => i.Blocked)) - .Number(fu => fu.Name(i => i.Discarded)) - .Number(fu => fu.Name(i => i.Limit)) - .Number(fu => fu.Name(i => i.TooBig)))) - .Object(ui => ui.Name(o => o.UsageHours.First()).Properties(p => p - .Date(fu => fu.Name(i => i.Date)) - .Number(fu => fu.Name(i => i.Total)) - .Number(fu => fu.Name(i => i.Blocked)) - .Number(fu => fu.Name(i => i.Discarded)) - .Number(fu => fu.Name(i => i.Limit)) - .Number(fu => fu.Name(i => i.TooBig)))); + .Object(o => o.Usage, ui => ui.Properties(p => p + .Date("date") + .FloatNumber("total") + .FloatNumber("blocked") + .FloatNumber("discarded") + .FloatNumber("limit") + .FloatNumber("too_big"))) + .Object(o => o.UsageHours, ui => ui.Properties(p => p + .Date("date") + .FloatNumber("total") + .FloatNumber("blocked") + .FloatNumber("discarded") + .FloatNumber("limit") + .FloatNumber("too_big"))); } } diff --git a/src/Exceptionless.Core/Repositories/Configuration/Indexes/SavedViewIndex.cs b/src/Exceptionless.Core/Repositories/Configuration/Indexes/SavedViewIndex.cs index 32d676f5ac..d0fbc703ed 100644 --- a/src/Exceptionless.Core/Repositories/Configuration/Indexes/SavedViewIndex.cs +++ b/src/Exceptionless.Core/Repositories/Configuration/Indexes/SavedViewIndex.cs @@ -1,6 +1,8 @@ +using Elastic.Clients.Elasticsearch.IndexManagement; +using Elastic.Clients.Elasticsearch.Mapping; +using Foundatio.Parsers.ElasticQueries.Extensions; using Foundatio.Repositories.Elasticsearch.Configuration; using Foundatio.Repositories.Elasticsearch.Extensions; -using Nest; namespace Exceptionless.Core.Repositories.Configuration; @@ -15,27 +17,28 @@ public SavedViewIndex(ExceptionlessElasticConfiguration configuration) : base(co _configuration = configuration; } - public override TypeMappingDescriptor ConfigureIndexMapping(TypeMappingDescriptor map) + public override void ConfigureIndexMapping(TypeMappingDescriptor map) { - return map - .Dynamic(false) + map + .Dynamic(DynamicMapping.False) .Properties(p => p .SetupDefaults() - .Keyword(f => f.Name(e => e.OrganizationId)) - .Keyword(f => f.Name(e => e.UserId)) - .Keyword(f => f.Name(e => e.CreatedByUserId)) - .Keyword(f => f.Name(e => e.UpdatedByUserId)) - .Text(f => f.Name(e => e.Name).Analyzer(KEYWORD_LOWERCASE_ANALYZER).AddKeywordField()) - .Keyword(f => f.Name(e => e.ViewType)) - .Number(f => f.Name(e => e.Version).Type(NumberType.Integer))); + .Keyword(e => e.OrganizationId) + .Keyword(e => e.UserId) + .Keyword(e => e.CreatedByUserId) + .Keyword(e => e.UpdatedByUserId) + .Text(e => e.Name, t => t.Analyzer(KEYWORD_LOWERCASE_ANALYZER).AddKeywordField()) + .Keyword(e => e.ViewType) + .IntegerNumber(e => e.Version)); } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - return base.ConfigureIndex(idx.Settings(s => s - .Analysis(d => d.Analyzers(b => b.Custom(KEYWORD_LOWERCASE_ANALYZER, c => c.Filters("lowercase").Tokenizer("keyword")))) + base.ConfigureIndex(idx); + idx.Settings(s => s + .Analysis(d => d.Analyzers(b => b.Custom(KEYWORD_LOWERCASE_ANALYZER, c => c.Filter("lowercase").Tokenizer("keyword")))) .NumberOfShards(_configuration.Options.NumberOfShards) .NumberOfReplicas(_configuration.Options.NumberOfReplicas) - .Priority(5))); + .Priority(5)); } } diff --git a/src/Exceptionless.Core/Repositories/Configuration/Indexes/StackIndex.cs b/src/Exceptionless.Core/Repositories/Configuration/Indexes/StackIndex.cs index 810b68e775..02b15bdd89 100644 --- a/src/Exceptionless.Core/Repositories/Configuration/Indexes/StackIndex.cs +++ b/src/Exceptionless.Core/Repositories/Configuration/Indexes/StackIndex.cs @@ -1,9 +1,10 @@ +using Elastic.Clients.Elasticsearch.IndexManagement; +using Elastic.Clients.Elasticsearch.Mapping; using Exceptionless.Core.Models; using Exceptionless.Core.Repositories.Queries; using Foundatio.Parsers.ElasticQueries; using Foundatio.Repositories.Elasticsearch.Configuration; using Foundatio.Repositories.Elasticsearch.Extensions; -using Nest; namespace Exceptionless.Core.Repositories.Configuration; @@ -21,50 +22,51 @@ public StackIndex(ExceptionlessElasticConfiguration configuration) : base(config _configuration = configuration; } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - return base.ConfigureIndex(idx.Settings(s => s - .Analysis(BuildAnalysis) + base.ConfigureIndex(idx); + idx.Settings(s => s + .Analysis(a => BuildAnalysis(a)) .NumberOfShards(_configuration.Options.NumberOfShards) .NumberOfReplicas(_configuration.Options.NumberOfReplicas) - .Priority(5))); + .Priority(5)); } - public override TypeMappingDescriptor ConfigureIndexMapping(TypeMappingDescriptor map) + public override void ConfigureIndexMapping(TypeMappingDescriptor map) { - return map - .Dynamic(false) + map + .Dynamic(DynamicMapping.False) .Properties(p => p .SetupDefaults() - .Keyword(f => f.Name(s => s.OrganizationId).IgnoreAbove(1024)) - .FieldAlias(a => a.Name(Alias.OrganizationId).Path(f => f.OrganizationId)) - .Keyword(f => f.Name(s => s.ProjectId).IgnoreAbove(1024)) - .FieldAlias(a => a.Name(Alias.ProjectId).Path(f => f.ProjectId)) - .Keyword(f => f.Name(s => s.Status)) - .Date(f => f.Name(s => s.SnoozeUntilUtc)) - .Keyword(f => f.Name(s => s.SignatureHash).IgnoreAbove(1024)) - .FieldAlias(a => a.Name(Alias.SignatureHash).Path(f => f.SignatureHash)) - .Keyword(f => f.Name(s => s.DuplicateSignature)) - .Keyword(f => f.Name(e => e.Type).IgnoreAbove(1024)) - .Date(f => f.Name(s => s.FirstOccurrence)) - .FieldAlias(a => a.Name(Alias.FirstOccurrence).Path(f => f.FirstOccurrence)) - .Date(f => f.Name(s => s.LastOccurrence)) - .FieldAlias(a => a.Name(Alias.LastOccurrence).Path(f => f.LastOccurrence)) - .Text(f => f.Name(s => s.Title)) - .Text(f => f.Name(s => s.Description)) - .Keyword(f => f.Name(s => s.Tags).IgnoreAbove(1024)) - .FieldAlias(a => a.Name(Alias.Tags).Path(f => f.Tags)) - .Keyword(f => f.Name(s => s.References).IgnoreAbove(1024)) - .FieldAlias(a => a.Name(Alias.References).Path(f => f.References)) - .Date(f => f.Name(s => s.DateFixed)) - .FieldAlias(a => a.Name(Alias.DateFixed).Path(f => f.DateFixed)) - .Boolean(f => f.Name(Alias.IsFixed)) - .Keyword(f => f.Name(s => s.FixedInVersion).IgnoreAbove(1024)) - .FieldAlias(a => a.Name(Alias.FixedInVersion).Path(f => f.FixedInVersion)) - .Boolean(f => f.Name(s => s.OccurrencesAreCritical)) - .FieldAlias(a => a.Name(Alias.OccurrencesAreCritical).Path(f => f.OccurrencesAreCritical)) - .Scalar(f => f.TotalOccurrences) - .FieldAlias(a => a.Name(Alias.TotalOccurrences).Path(f => f.TotalOccurrences)) + .Keyword(e => e.OrganizationId, k => k.IgnoreAbove(1024)) + .FieldAlias(Alias.OrganizationId, a => a.Path(f => f.OrganizationId)) + .Keyword(e => e.ProjectId, k => k.IgnoreAbove(1024)) + .FieldAlias(Alias.ProjectId, a => a.Path(f => f.ProjectId)) + .Keyword(e => e.Status) + .Date(e => e.SnoozeUntilUtc) + .Keyword(e => e.SignatureHash, k => k.IgnoreAbove(1024)) + .FieldAlias(Alias.SignatureHash, a => a.Path(f => f.SignatureHash)) + .Keyword(e => e.DuplicateSignature) + .Keyword(e => e.Type, k => k.IgnoreAbove(1024)) + .Date(e => e.FirstOccurrence) + .FieldAlias(Alias.FirstOccurrence, a => a.Path(f => f.FirstOccurrence)) + .Date(e => e.LastOccurrence) + .FieldAlias(Alias.LastOccurrence, a => a.Path(f => f.LastOccurrence)) + .Text(e => e.Title) + .Text(e => e.Description) + .Keyword(e => e.Tags, k => k.IgnoreAbove(1024)) + .FieldAlias(Alias.Tags, a => a.Path(f => f.Tags)) + .Keyword(e => e.References, k => k.IgnoreAbove(1024)) + .FieldAlias(Alias.References, a => a.Path(f => f.References)) + .Date(e => e.DateFixed) + .FieldAlias(Alias.DateFixed, a => a.Path(f => f.DateFixed)) + .Boolean(Alias.IsFixed) + .Keyword(e => e.FixedInVersion, k => k.IgnoreAbove(1024)) + .FieldAlias(Alias.FixedInVersion, a => a.Path(f => f.FixedInVersion)) + .Boolean(e => e.OccurrencesAreCritical) + .FieldAlias(Alias.OccurrencesAreCritical, a => a.Path(f => f.OccurrencesAreCritical)) + .IntegerNumber(e => e.TotalOccurrences) + .FieldAlias(Alias.TotalOccurrences, a => a.Path(f => f.TotalOccurrences)) ); } @@ -79,12 +81,12 @@ protected override void ConfigureQueryParser(ElasticQueryParserConfiguration con }); } - private AnalysisDescriptor BuildAnalysis(AnalysisDescriptor ad) + private void BuildAnalysis(IndexSettingsAnalysisDescriptor ad) { - return ad.Analyzers(a => a + ad.Analyzers(a => a .Pattern(COMMA_WHITESPACE_ANALYZER, p => p.Pattern(@"[,\s]+")) - .Custom(STANDARDPLUS_ANALYZER, c => c.Filters("lowercase", "stop", "unique").Tokenizer(COMMA_WHITESPACE_TOKENIZER)) - .Custom(WHITESPACE_LOWERCASE_ANALYZER, c => c.Filters("lowercase").Tokenizer("whitespace"))) + .Custom(STANDARDPLUS_ANALYZER, c => c.Filter("lowercase", "stop", "unique").Tokenizer(COMMA_WHITESPACE_TOKENIZER)) + .Custom(WHITESPACE_LOWERCASE_ANALYZER, c => c.Filter("lowercase").Tokenizer("whitespace"))) .Tokenizers(t => t .Pattern(COMMA_WHITESPACE_TOKENIZER, p => p.Pattern(@"[,\s]+"))); } diff --git a/src/Exceptionless.Core/Repositories/Configuration/Indexes/TokenIndex.cs b/src/Exceptionless.Core/Repositories/Configuration/Indexes/TokenIndex.cs index c5e86ef488..f512d73bd7 100644 --- a/src/Exceptionless.Core/Repositories/Configuration/Indexes/TokenIndex.cs +++ b/src/Exceptionless.Core/Repositories/Configuration/Indexes/TokenIndex.cs @@ -1,6 +1,7 @@ -using Foundatio.Repositories.Elasticsearch.Configuration; +using Elastic.Clients.Elasticsearch.IndexManagement; +using Elastic.Clients.Elasticsearch.Mapping; +using Foundatio.Repositories.Elasticsearch.Configuration; using Foundatio.Repositories.Elasticsearch.Extensions; -using Nest; namespace Exceptionless.Core.Repositories.Configuration; @@ -14,31 +15,32 @@ public TokenIndex(ExceptionlessElasticConfiguration configuration) : base(config _configuration = configuration; } - public override TypeMappingDescriptor ConfigureIndexMapping(TypeMappingDescriptor map) + public override void ConfigureIndexMapping(TypeMappingDescriptor map) { - return map - .Dynamic(false) + map + .Dynamic(DynamicMapping.False) .Properties(p => p .SetupDefaults() - .Date(f => f.Name(e => e.ExpiresUtc)) - .Keyword(f => f.Name(e => e.OrganizationId)) - .Keyword(f => f.Name(e => e.ProjectId)) - .Keyword(f => f.Name(e => e.DefaultProjectId)) - .Keyword(f => f.Name(e => e.UserId)) - .Keyword(f => f.Name(u => u.CreatedBy)) - .Keyword(f => f.Name(e => e.Refresh)) - .Keyword(f => f.Name(e => e.Scopes)) - .Boolean(f => f.Name(e => e.IsDisabled)) - .Boolean(f => f.Name(e => e.IsSuspended)) - .Number(f => f.Name(e => e.Type).Type(NumberType.Byte))); + .Date(e => e.ExpiresUtc) + .Keyword(e => e.OrganizationId) + .Keyword(e => e.ProjectId) + .Keyword(e => e.DefaultProjectId) + .Keyword(e => e.UserId) + .Keyword(e => e.CreatedBy) + .Keyword(e => e.Refresh) + .Keyword(e => e.Scopes) + .Boolean(e => e.IsDisabled) + .Boolean(e => e.IsSuspended) + .ByteNumber(e => e.Type)); } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - return base.ConfigureIndex(idx.Settings(s => s - .Analysis(d => d.Analyzers(b => b.Custom(KEYWORD_LOWERCASE_ANALYZER, c => c.Filters("lowercase").Tokenizer("keyword")))) + base.ConfigureIndex(idx); + idx.Settings(s => s + .Analysis(d => d.Analyzers(b => b.Custom(KEYWORD_LOWERCASE_ANALYZER, c => c.Filter("lowercase").Tokenizer("keyword")))) .NumberOfShards(_configuration.Options.NumberOfShards) .NumberOfReplicas(_configuration.Options.NumberOfReplicas) - .Priority(10))); + .Priority(10)); } } diff --git a/src/Exceptionless.Core/Repositories/Configuration/Indexes/UserIndex.cs b/src/Exceptionless.Core/Repositories/Configuration/Indexes/UserIndex.cs index 0bfb36a32c..d777ac2013 100644 --- a/src/Exceptionless.Core/Repositories/Configuration/Indexes/UserIndex.cs +++ b/src/Exceptionless.Core/Repositories/Configuration/Indexes/UserIndex.cs @@ -1,7 +1,9 @@ +using Elastic.Clients.Elasticsearch.IndexManagement; +using Elastic.Clients.Elasticsearch.Mapping; using Exceptionless.Core.Models; +using Foundatio.Parsers.ElasticQueries.Extensions; using Foundatio.Repositories.Elasticsearch.Configuration; using Foundatio.Repositories.Elasticsearch.Extensions; -using Nest; namespace Exceptionless.Core.Repositories.Configuration; @@ -15,34 +17,35 @@ public UserIndex(ExceptionlessElasticConfiguration configuration) : base(configu _configuration = configuration; } - public override TypeMappingDescriptor ConfigureIndexMapping(TypeMappingDescriptor map) + public override void ConfigureIndexMapping(TypeMappingDescriptor map) { - return map - .Dynamic(false) + map + .Dynamic(DynamicMapping.False) .Properties(p => p .SetupDefaults() - .Keyword(f => f.Name(e => e.OrganizationIds)) - .Text(f => f.Name(u => u.FullName).AddKeywordField()) - .Text(f => f.Name(u => u.EmailAddress).Analyzer(KEYWORD_LOWERCASE_ANALYZER).AddKeywordField()) - .Boolean(f => f.Name(u => u.IsEmailAddressVerified)) - .Keyword(f => f.Name(u => u.VerifyEmailAddressToken)) - .Date(f => f.Name(u => u.VerifyEmailAddressTokenExpiration)) - .Keyword(f => f.Name(u => u.PasswordResetToken)) - .Date(f => f.Name(u => u.PasswordResetTokenExpiration)) - .Keyword(f => f.Name(u => u.Roles)) - .Object(f => f.Name(o => o.OAuthAccounts.First()).Properties(mp => mp - .Keyword(fu => fu.Name(m => m.Provider)) - .Keyword(fu => fu.Name(m => m.ProviderUserId)) - .Keyword(fu => fu.Name(m => m.Username)))) + .Keyword(e => e.OrganizationIds) + .Text(e => e.FullName, t => t.AddKeywordField()) + .Text(e => e.EmailAddress, t => t.Analyzer(KEYWORD_LOWERCASE_ANALYZER).AddKeywordField()) + .Boolean(e => e.IsEmailAddressVerified) + .Keyword(e => e.VerifyEmailAddressToken) + .Date(e => e.VerifyEmailAddressTokenExpiration) + .Keyword(e => e.PasswordResetToken) + .Date(e => e.PasswordResetTokenExpiration) + .Keyword(e => e.Roles) + .Object(e => e.OAuthAccounts, o => o.Properties(mp => mp + .Keyword("provider") + .Keyword("provider_user_id") + .Keyword("username"))) ); } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - return base.ConfigureIndex(idx.Settings(s => s - .Analysis(d => d.Analyzers(b => b.Custom(KEYWORD_LOWERCASE_ANALYZER, c => c.Filters("lowercase").Tokenizer("keyword")))) + base.ConfigureIndex(idx); + idx.Settings(s => s + .Analysis(d => d.Analyzers(b => b.Custom(KEYWORD_LOWERCASE_ANALYZER, c => c.Filter("lowercase").Tokenizer("keyword")))) .NumberOfShards(_configuration.Options.NumberOfShards) .NumberOfReplicas(_configuration.Options.NumberOfReplicas) - .Priority(5))); + .Priority(5)); } } diff --git a/src/Exceptionless.Core/Repositories/Configuration/Indexes/WebHookIndex.cs b/src/Exceptionless.Core/Repositories/Configuration/Indexes/WebHookIndex.cs index bd6f85da2e..d7fb394700 100644 --- a/src/Exceptionless.Core/Repositories/Configuration/Indexes/WebHookIndex.cs +++ b/src/Exceptionless.Core/Repositories/Configuration/Indexes/WebHookIndex.cs @@ -1,7 +1,8 @@ +using Elastic.Clients.Elasticsearch.IndexManagement; +using Elastic.Clients.Elasticsearch.Mapping; using Exceptionless.Core.Models; using Foundatio.Repositories.Elasticsearch.Configuration; using Foundatio.Repositories.Elasticsearch.Extensions; -using Nest; namespace Exceptionless.Core.Repositories.Configuration; @@ -14,25 +15,26 @@ public WebHookIndex(ExceptionlessElasticConfiguration configuration) : base(conf _configuration = configuration; } - public override TypeMappingDescriptor ConfigureIndexMapping(TypeMappingDescriptor map) + public override void ConfigureIndexMapping(TypeMappingDescriptor map) { - return map - .Dynamic(false) + map + .Dynamic(DynamicMapping.False) .Properties(p => p .SetupDefaults() - .Keyword(f => f.Name(e => e.OrganizationId)) - .Keyword(f => f.Name(e => e.ProjectId)) - .Keyword(f => f.Name(e => e.Url)) - .Keyword(f => f.Name(e => e.EventTypes)) - .Boolean(f => f.Name(e => e.IsEnabled)) + .Keyword(e => e.OrganizationId) + .Keyword(e => e.ProjectId) + .Keyword(e => e.Url) + .Keyword(e => e.EventTypes) + .Boolean(e => e.IsEnabled) ); } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - return base.ConfigureIndex(idx.Settings(s => s + base.ConfigureIndex(idx); + idx.Settings(s => s .NumberOfShards(_configuration.Options.NumberOfShards) .NumberOfReplicas(_configuration.Options.NumberOfReplicas) - .Priority(5))); + .Priority(5)); } } diff --git a/src/Exceptionless.Core/Repositories/EventRepository.cs b/src/Exceptionless.Core/Repositories/EventRepository.cs index fbc18d9f4d..3fba37a69d 100644 --- a/src/Exceptionless.Core/Repositories/EventRepository.cs +++ b/src/Exceptionless.Core/Repositories/EventRepository.cs @@ -1,11 +1,11 @@ -using Exceptionless.Core.Models; +using Elastic.Clients.Elasticsearch.QueryDsl; +using Exceptionless.Core.Models; using Exceptionless.Core.Repositories.Configuration; using Exceptionless.Core.Repositories.Queries; using Exceptionless.Core.Validation; using Exceptionless.DateTimeExtensions; using Foundatio.Repositories; using Foundatio.Repositories.Models; -using Nest; namespace Exceptionless.Core.Repositories; @@ -22,7 +22,7 @@ public EventRepository(ExceptionlessElasticConfiguration configuration, AppOptio BatchNotifications = true; DefaultPipeline = "events-pipeline"; - AddDefaultExclude(e => e.Idx!); + AddDefaultExclude(e => e.Idx); // copy to fields AddDefaultExclude(EventIndex.Alias.IpAddress); AddDefaultExclude(EventIndex.Alias.OperatingSystem); @@ -33,11 +33,14 @@ public EventRepository(ExceptionlessElasticConfiguration configuration, AppOptio public Task> GetOpenSessionsAsync(DateTime createdBeforeUtc, CommandOptionsDescriptor? options = null) { - var filter = Query.Term(e => e.Type, Event.KnownTypes.Session) && !Query.Exists(f => f.Field(e => e.Idx![Event.KnownDataKeys.SessionEnd + "-d"])); + var query = new RepositoryQuery() + .FieldEquals(e => e.Type, Event.KnownTypes.Session) + .ElasticFilter(new BoolQuery { MustNot = [new ExistsQuery { Field = $"idx.{Event.KnownDataKeys.SessionEnd}-d" }] }); + if (createdBeforeUtc.Ticks > 0) - filter &= Query.DateRange(r => r.Field(e => e.Date).LessThanOrEquals(createdBeforeUtc)); + query = query.DateRange(null, createdBeforeUtc, (PersistentEvent e) => e.Date); // No lower bound, upper bound is exclusive - return FindAsync(q => q.ElasticFilter(filter).SortDescending(e => e.Date), options); + return FindAsync(q => query.SortDescending(e => e.Date), options); } /// @@ -64,9 +67,9 @@ public Task RemoveAllAsync(string organizationId, string? clientIpAddress, if (utcStart.HasValue && utcEnd.HasValue) query = query.DateRange(utcStart, utcEnd, InferField(e => e.Date)).Index(utcStart, utcEnd); else if (utcEnd.HasValue) - query = query.ElasticFilter(Query.DateRange(r => r.Field(e => e.Date).LessThan(utcEnd))); + query = query.DateRange(null, utcEnd, (PersistentEvent e) => e.Date); else if (utcStart.HasValue) - query = query.ElasticFilter(Query.DateRange(r => r.Field(e => e.Date).GreaterThan(utcStart))); + query = query.DateRange(utcStart, null, (PersistentEvent e) => e.Date); if (!String.IsNullOrEmpty(clientIpAddress)) query = query.FieldEquals(EventIndex.Alias.IpAddress, clientIpAddress); @@ -76,8 +79,7 @@ public Task RemoveAllAsync(string organizationId, string? clientIpAddress, public Task> GetByReferenceIdAsync(string projectId, string referenceId) { - var filter = Query.Term(e => e.ReferenceId, referenceId); - return FindAsync(q => q.Project(projectId).ElasticFilter(filter).SortDescending(e => e.Date), o => o.PageLimit(10)); + return FindAsync(q => q.Project(projectId).FieldEquals(e => e.ReferenceId, referenceId).SortDescending(e => e.Date), o => o.PageLimit(10)); } public async Task GetPreviousAndNextEventIdsAsync(PersistentEvent ev, AppFilter? systemFilter = null, DateTime? utcStart = null, DateTime? utcEnd = null) @@ -113,8 +115,8 @@ public async Task GetPreviousAndNextEventIdsAsync( .SortDescending(e => e.Date) .Include(e => e.Id, e => e.Date) .AppFilter(systemFilter) - .ElasticFilter(!Query.Ids(ids => ids.Values(ev.Id))) - .FilterExpression(String.Concat(EventIndex.Alias.StackId, ":", ev.StackId)) + .Stack(ev.StackId) + .ExcludedId(ev.Id) .EnforceEventStackFilter(false), o => o.PageLimit(10)); if (results.Total == 0) @@ -153,8 +155,8 @@ public async Task GetPreviousAndNextEventIdsAsync( .SortAscending(e => e.Date) .Include(e => e.Id, e => e.Date) .AppFilter(systemFilter) - .ElasticFilter(!Query.Ids(ids => ids.Values(ev.Id))) - .FilterExpression(String.Concat(EventIndex.Alias.StackId, ":", ev.StackId)) + .Stack(ev.StackId) + .ExcludedId(ev.Id) .EnforceEventStackFilter(false), o => o.PageLimit(10)); if (results.Total == 0) diff --git a/src/Exceptionless.Core/Repositories/OrganizationRepository.cs b/src/Exceptionless.Core/Repositories/OrganizationRepository.cs index f41d80bc52..7ed3d1cfca 100644 --- a/src/Exceptionless.Core/Repositories/OrganizationRepository.cs +++ b/src/Exceptionless.Core/Repositories/OrganizationRepository.cs @@ -7,7 +7,6 @@ using Exceptionless.Core.Validation; using Foundatio.Repositories; using Foundatio.Repositories.Models; -using Nest; namespace Exceptionless.Core.Repositories; @@ -34,8 +33,7 @@ private void OnDocumentsChanging(object sender, DocumentsChangeEventArgs.Term(f => f.Field(o => o.Invites.First().Token).Value(token)); - var hit = await FindOneAsync(q => q.ElasticFilter(filter)); + var hit = await FindOneAsync(q => q.FieldEquals(o => o.Invites.First().Token, token)); return hit?.Document; } @@ -43,8 +41,7 @@ private void OnDocumentsChanging(object sender, DocumentsChangeEventArgs.Term(f => f.Field(o => o.StripeCustomerId).Value(customerId)); - var hit = await FindOneAsync(q => q.ElasticFilter(filter)); + var hit = await FindOneAsync(q => q.FieldEquals(o => o.StripeCustomerId, customerId)); return hit?.Document; } @@ -54,53 +51,59 @@ public Task> GetByFilterAsync(AppFilter systemFilter, .AppFilter(systemFilter) .FilterExpression(userFilter); - query = !String.IsNullOrEmpty(sort) ? query.SortExpression(sort) : query.SortAscending(o => o.Name.Suffix("keyword")); + query = !String.IsNullOrEmpty(sort) ? query.SortExpression(sort) : query.SortAscending(o => o.Name); return FindAsync(q => query, options); } public Task> GetByCriteriaAsync(string? criteria, CommandOptionsDescriptor options, OrganizationSortBy sortBy, bool? paid = null, bool? suspended = null) { - var filter = Query.MatchAll(); + var query = new RepositoryQuery(); + if (!String.IsNullOrWhiteSpace(criteria)) - filter &= (Query.Term(o => o.Id, criteria) || Query.Term(o => o.Name, criteria)); + query.FieldOr(g => g + .FieldEquals(o => o.Id, criteria) + .FieldEquals(o => o.Name, criteria)); if (paid.HasValue) { if (paid.Value) - filter &= !Query.Term(o => o.PlanId, _plans.FreePlan.Id); + query.FieldNotEquals(o => o.PlanId, _plans.FreePlan.Id); else - filter &= Query.Term(o => o.PlanId, _plans.FreePlan.Id); + query.FieldEquals(o => o.PlanId, _plans.FreePlan.Id); } if (suspended.HasValue) { if (suspended.Value) - filter &= (!Query.Term(o => o.BillingStatus, BillingStatus.Active) && - !Query.Term(o => o.BillingStatus, BillingStatus.Trialing) && - !Query.Term(o => o.BillingStatus, BillingStatus.Canceled) - ) || Query.Term(o => o.IsSuspended, true); + query.FieldOr(g => g + .FieldNot(n => n + .FieldOr(o => o + .FieldEquals(o => o.BillingStatus, (int)BillingStatus.Active) + .FieldEquals(o => o.BillingStatus, (int)BillingStatus.Trialing) + .FieldEquals(o => o.BillingStatus, (int)BillingStatus.Canceled))) + .FieldEquals(o => o.IsSuspended, true)); else - filter &= ( - Query.Term(o => o.BillingStatus, BillingStatus.Active) && - Query.Term(o => o.BillingStatus, BillingStatus.Trialing) && - Query.Term(o => o.BillingStatus, BillingStatus.Canceled) - ) || Query.Term(o => o.IsSuspended, false); + query.FieldAnd(g => g + .FieldOr(o => o + .FieldEquals(o => o.BillingStatus, (int)BillingStatus.Active) + .FieldEquals(o => o.BillingStatus, (int)BillingStatus.Trialing) + .FieldEquals(o => o.BillingStatus, (int)BillingStatus.Canceled)) + .FieldEquals(o => o.IsSuspended, false)); } - var query = new RepositoryQuery().ElasticFilter(filter); switch (sortBy) { case OrganizationSortBy.Newest: query.SortDescending((Organization o) => o.Id); break; case OrganizationSortBy.Subscribed: - query.SortDescending((Organization o) => o.SubscribeDate!); + query.SortDescending((Organization o) => o.SubscribeDate); break; // case OrganizationSortBy.MostActive: // query.WithSortDescending((Organization o) => o.TotalEventCount); // break; default: - query.SortAscending(o => o.Name.Suffix("keyword")); + query.SortAscending((Organization o) => o.Name); break; } diff --git a/src/Exceptionless.Core/Repositories/ProjectRepository.cs b/src/Exceptionless.Core/Repositories/ProjectRepository.cs index e338277851..afcdcd43e3 100644 --- a/src/Exceptionless.Core/Repositories/ProjectRepository.cs +++ b/src/Exceptionless.Core/Repositories/ProjectRepository.cs @@ -6,7 +6,6 @@ using Foundatio.Repositories; using Foundatio.Repositories.Models; using Foundatio.Repositories.Options; -using Nest; namespace Exceptionless.Core.Repositories; @@ -59,7 +58,7 @@ public Task> GetByOrganizationIdsAsync(ICollection if (organizationIds.Count == 0) return Task.FromResult(new FindResults()); - return FindAsync(q => q.Organization(organizationIds).SortAscending(p => p.Name.Suffix("keyword")), options); + return FindAsync(q => q.Organization(organizationIds).SortAscending(p => p.Name), options); } public Task> GetByFilterAsync(AppFilter systemFilter, string? userFilter, string? sort, CommandOptionsDescriptor? options = null) @@ -68,14 +67,16 @@ public Task> GetByFilterAsync(AppFilter systemFilter, strin .AppFilter(systemFilter) .FilterExpression(userFilter); - query = !String.IsNullOrEmpty(sort) ? query.SortExpression(sort) : query.SortAscending(p => p.Name.Suffix("keyword")); + query = !String.IsNullOrEmpty(sort) ? query.SortExpression(sort) : query.SortAscending(p => p.Name); return FindAsync(q => query, options); } public Task> GetByNextSummaryNotificationOffsetAsync(byte hourToSendNotificationsAfterUtcMidnight, int limit = 50) { - var filter = Query.Range(r => r.Field(o => o.NextSummaryEndOfDayTicks).LessThan(_timeProvider.GetUtcNow().UtcDateTime.Ticks - (TimeSpan.TicksPerHour * hourToSendNotificationsAfterUtcMidnight))); - return FindAsync(q => q.ElasticFilter(filter).SortAscending(p => p.OrganizationId), o => o.SearchAfterPaging().PageLimit(limit)); + long threshold = _timeProvider.GetUtcNow().UtcDateTime.Ticks - (TimeSpan.TicksPerHour * hourToSendNotificationsAfterUtcMidnight); + return FindAsync(q => q + .FieldLessThan(p => p.NextSummaryEndOfDayTicks, threshold) + .SortAscending(p => p.OrganizationId), o => o.SearchAfterPaging().PageLimit(limit)); } public async Task IncrementNextSummaryEndOfDayTicksAsync(IReadOnlyCollection projects) diff --git a/src/Exceptionless.Core/Repositories/Queries/AppFilterQuery.cs b/src/Exceptionless.Core/Repositories/Queries/AppFilterQuery.cs index 4af8b8ebb1..47167278c9 100644 --- a/src/Exceptionless.Core/Repositories/Queries/AppFilterQuery.cs +++ b/src/Exceptionless.Core/Repositories/Queries/AppFilterQuery.cs @@ -1,3 +1,4 @@ +using Elastic.Clients.Elasticsearch.QueryDsl; using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Repositories.Configuration; @@ -8,7 +9,6 @@ using Foundatio.Repositories.Elasticsearch.Configuration; using Foundatio.Repositories.Elasticsearch.Queries.Builders; using Foundatio.Repositories.Options; -using Nest; namespace Exceptionless.Core.Repositories { @@ -107,83 +107,86 @@ public AppFilterQueryBuilder(AppOptions options, TimeProvider timeProvider) return Task.CompletedTask; var allowedOrganizations = sfq.Organizations.Where(o => o.HasPremiumFeatures || (!o.HasPremiumFeatures && !sfq.UsesPremiumFeatures)).ToList(); + bool isOrganizationIndex = typeof(T) == typeof(Organization); + string organizationIdFieldName = isOrganizationIndex ? "id" : _organizationIdFieldName; if (allowedOrganizations.Count == 0) { - ctx.Filter &= Query.Term(_organizationIdFieldName, "none"); + ctx.Filter &= new TermQuery { Field = organizationIdFieldName, Value = "none" }; return Task.CompletedTask; } var index = ctx.Options.GetElasticIndex(); - bool shouldApplyRetentionFilter = ShouldApplyRetentionFilter(index, ctx); - string? field = shouldApplyRetentionFilter ? GetDateField(index) : null; + bool shouldApplyRetentionFilter = index is not null && ShouldApplyRetentionFilter(index, ctx); + string? field = shouldApplyRetentionFilter ? GetDateField(index!) : null; - if (sfq.Stack is not null) + if (!isOrganizationIndex && sfq.Stack is not null) { string stackIdFieldName = typeof(T) == typeof(Stack) ? "id" : _stackIdFieldName; - var organization = allowedOrganizations.SingleOrDefault(o => o.Id == sfq.Stack.OrganizationId); + var organization = allowedOrganizations.SingleOrDefault(o => String.Equals(o.Id, sfq.Stack.OrganizationId, StringComparison.Ordinal)); if (organization is not null) { if (shouldApplyRetentionFilter) - ctx.Filter &= (Query.Term(stackIdFieldName, sfq.Stack.Id) && GetRetentionFilter(field, organization, _options.MaximumRetentionDays, sfq.Stack.FirstOccurrence)); + ctx.Filter &= new TermQuery { Field = stackIdFieldName, Value = sfq.Stack.Id } & GetRetentionFilter(field, organization, _options.MaximumRetentionDays, sfq.Stack.FirstOccurrence); else { - ctx.Filter &= Query.Term(stackIdFieldName, sfq.Stack.Id); + ctx.Filter &= new TermQuery { Field = stackIdFieldName, Value = sfq.Stack.Id }; } } else { - ctx.Filter &= Query.Term(stackIdFieldName, "none"); + ctx.Filter &= new TermQuery { Field = stackIdFieldName, Value = "none" }; } return Task.CompletedTask; } - QueryContainer? container = null; - if (sfq.Projects?.Count > 0) + Query? container = null; + if (!isOrganizationIndex && sfq.Projects?.Count > 0) { - var allowedProjects = sfq.Projects.ToDictionary(p => p, p => allowedOrganizations.SingleOrDefault(o => o.Id == p.OrganizationId)).Where(kvp => kvp.Value is not null).ToList(); + var allowedProjects = sfq.Projects.ToDictionary(p => p, p => allowedOrganizations.SingleOrDefault(o => String.Equals(o.Id, p.OrganizationId, StringComparison.Ordinal))).Where(kvp => kvp.Value is not null).ToList(); if (allowedProjects.Count > 0) { foreach (var project in allowedProjects) { + Query termQuery = new TermQuery { Field = _projectIdFieldName, Value = project.Key.Id }; if (shouldApplyRetentionFilter) - container |= (Query.Term(_projectIdFieldName, project.Key.Id) && GetRetentionFilter(field, project.Value!, _options.MaximumRetentionDays, project.Key.CreatedUtc.SafeSubtract(TimeSpan.FromDays(3)))); - else - container |= Query.Term(_projectIdFieldName, project.Key.Id); + termQuery &= GetRetentionFilter(field, project.Value!, _options.MaximumRetentionDays, project.Key.CreatedUtc.SafeSubtract(TimeSpan.FromDays(3))); + container = container is not null ? container | termQuery : termQuery; } - ctx.Filter &= container; + if (container is not null) + ctx.Filter &= container; return Task.CompletedTask; } - ctx.Filter &= (Query.Term(_projectIdFieldName, "none")); + ctx.Filter &= new TermQuery { Field = _projectIdFieldName, Value = "none" }; return Task.CompletedTask; } - string organizationIdFieldName = typeof(T) == typeof(Organization) ? "id" : _organizationIdFieldName; foreach (var organization in allowedOrganizations) { + Query termQuery = new TermQuery { Field = organizationIdFieldName, Value = organization.Id }; if (shouldApplyRetentionFilter) - container |= (Query.Term(organizationIdFieldName, organization.Id) && GetRetentionFilter(field, organization, _options.MaximumRetentionDays)); - else - container |= Query.Term(organizationIdFieldName, organization.Id); + termQuery &= GetRetentionFilter(field, organization, _options.MaximumRetentionDays); + container = container is not null ? container | termQuery : termQuery; } - ctx.Filter &= container; + if (container is not null) + ctx.Filter &= container; return Task.CompletedTask; } - private QueryContainer GetRetentionFilter(string? field, Organization organization, int maximumRetentionDays, DateTime? oldestPossibleEventAge = null) where T : class, new() + private Query GetRetentionFilter(string? field, Organization organization, int maximumRetentionDays, DateTime? oldestPossibleEventAge = null) { if (field is null) throw new ArgumentNullException(nameof(field), "Retention field not specified for this index"); var retentionDate = organization.GetRetentionUtcCutoff(maximumRetentionDays, oldestPossibleEventAge, _timeProvider); double retentionDays = Math.Max(Math.Round(Math.Abs(_timeProvider.GetUtcNow().UtcDateTime.Subtract(retentionDate).TotalDays), MidpointRounding.AwayFromZero), 1); - return Query.DateRange(r => r.Field(field).GreaterThanOrEquals($"now/d-{(int)retentionDays}d").LessThanOrEquals("now/d+1d")); + return new DateRangeQuery { Field = field, Gte = $"now/d-{(int)retentionDays}d", Lte = "now/d+1d" }; } - private static bool ShouldApplyRetentionFilter(IIndex? index, QueryBuilderContext ctx) where T : class, new() + private static bool ShouldApplyRetentionFilter(IIndex index, QueryBuilderContext ctx) where T : class, new() { ArgumentNullException.ThrowIfNull(index); @@ -197,7 +200,7 @@ public AppFilterQueryBuilder(AppOptions options, TimeProvider timeProvider) return false; } - private string? GetDateField(IIndex? index) + private string? GetDateField(IIndex index) { ArgumentNullException.ThrowIfNull(index); diff --git a/src/Exceptionless.Core/Repositories/Queries/EventStackFilterQuery.cs b/src/Exceptionless.Core/Repositories/Queries/EventStackFilterQuery.cs index 4d78edd8e1..df6202a7bd 100644 --- a/src/Exceptionless.Core/Repositories/Queries/EventStackFilterQuery.cs +++ b/src/Exceptionless.Core/Repositories/Queries/EventStackFilterQuery.cs @@ -1,3 +1,4 @@ +using Elastic.Clients.Elasticsearch; using Exceptionless.Core.Models; using Exceptionless.Core.Repositories.Base; using Exceptionless.Core.Repositories.Options; @@ -11,7 +12,6 @@ using Foundatio.Repositories.Models; using Foundatio.Repositories.Options; using Microsoft.Extensions.Logging; -using Nest; using DateRange = Foundatio.Repositories.DateRange; namespace Exceptionless.Core.Repositories diff --git a/src/Exceptionless.Core/Repositories/Queries/OrganizationQuery.cs b/src/Exceptionless.Core/Repositories/Queries/OrganizationQuery.cs index bda527bb58..7fc1089832 100644 --- a/src/Exceptionless.Core/Repositories/Queries/OrganizationQuery.cs +++ b/src/Exceptionless.Core/Repositories/Queries/OrganizationQuery.cs @@ -1,10 +1,11 @@ -using Exceptionless.Core.Extensions; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.QueryDsl; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Repositories.Options; using Foundatio.Repositories; using Foundatio.Repositories.Elasticsearch.Queries.Builders; using Foundatio.Repositories.Options; -using Nest; namespace Exceptionless.Core.Repositories { @@ -54,9 +55,9 @@ public class OrganizationQueryBuilder : IElasticQueryBuilder return Task.CompletedTask; if (organizationIds.Count == 1) - ctx.Filter &= Query.Term(_organizationIdFieldName, organizationIds.Single()); + ctx.Filter &= new TermQuery { Field = _organizationIdFieldName, Value = organizationIds.Single() }; else - ctx.Filter &= Query.Terms(d => d.Field(_organizationIdFieldName).Terms(organizationIds)); + ctx.Filter &= new TermsQuery { Field = _organizationIdFieldName, Terms = new TermsQueryField(organizationIds.Select(id => (FieldValue)id).ToList()) }; return Task.CompletedTask; } diff --git a/src/Exceptionless.Core/Repositories/Queries/ProjectQuery.cs b/src/Exceptionless.Core/Repositories/Queries/ProjectQuery.cs index 9be2eb330d..46a27e7628 100644 --- a/src/Exceptionless.Core/Repositories/Queries/ProjectQuery.cs +++ b/src/Exceptionless.Core/Repositories/Queries/ProjectQuery.cs @@ -1,10 +1,11 @@ -using Exceptionless.Core.Extensions; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.QueryDsl; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Repositories.Options; using Foundatio.Repositories; using Foundatio.Repositories.Elasticsearch.Queries.Builders; using Foundatio.Repositories.Options; -using Nest; namespace Exceptionless.Core.Repositories { @@ -48,9 +49,9 @@ public class ProjectQueryBuilder : IElasticQueryBuilder return Task.CompletedTask; if (projectIds.Count == 1) - ctx.Filter &= Query.Term(_projectIdFieldName, projectIds.Single()); + ctx.Filter &= new TermQuery { Field = _projectIdFieldName, Value = projectIds.Single() }; else - ctx.Filter &= Query.Terms(d => d.Field(_projectIdFieldName).Terms(projectIds)); + ctx.Filter &= new TermsQuery { Field = _projectIdFieldName, Terms = new TermsQueryField(projectIds.Select(id => (FieldValue)id).ToList()) }; return Task.CompletedTask; } diff --git a/src/Exceptionless.Core/Repositories/Queries/StackQuery.cs b/src/Exceptionless.Core/Repositories/Queries/StackQuery.cs index db49aeb7cf..1d893d11e6 100644 --- a/src/Exceptionless.Core/Repositories/Queries/StackQuery.cs +++ b/src/Exceptionless.Core/Repositories/Queries/StackQuery.cs @@ -1,10 +1,12 @@ -using Exceptionless.Core.Extensions; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.QueryDsl; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Repositories.Options; using Foundatio.Repositories; using Foundatio.Repositories.Elasticsearch.Queries.Builders; +using Foundatio.Repositories.Elasticsearch.Utility; using Foundatio.Repositories.Options; -using Nest; namespace Exceptionless.Core.Repositories { @@ -55,22 +57,21 @@ namespace Exceptionless.Core.Repositories.Queries { public class StackQueryBuilder : IElasticQueryBuilder { - private readonly string _stackIdFieldName = nameof(IOwnedByStack.StackId).ToLowerUnderscoredWords(); + private static readonly Field StackIdField = nameof(IOwnedByStack.StackId).ToLowerUnderscoredWords(); public Task BuildAsync(QueryBuilderContext ctx) where T : class, new() { var stackIds = ctx.Source.GetStacks(); - var excludedStackIds = ctx.Source.GetExcludedStacks(); - if (stackIds.Count == 1) - ctx.Filter &= Query.Term(_stackIdFieldName, stackIds.Single()); + ctx.Filter &= new TermQuery { Field = StackIdField, Value = stackIds.Single() }; else if (stackIds.Count > 1) - ctx.Filter &= Query.Terms(d => d.Field(_stackIdFieldName).Terms(stackIds)); + ctx.Filter &= new TermsQuery { Field = StackIdField, Terms = new TermsQueryField(stackIds.Select(FieldValueHelper.ToFieldValue).ToList()) }; + var excludedStackIds = ctx.Source.GetExcludedStacks(); if (excludedStackIds.Count == 1) - ctx.Filter &= Query.Bool(b => b.MustNot(Query.Term(_stackIdFieldName, excludedStackIds.Single()))); + ctx.Filter &= new BoolQuery { MustNot = [new TermQuery { Field = StackIdField, Value = excludedStackIds.Single() }] }; else if (excludedStackIds.Count > 1) - ctx.Filter &= Query.Bool(b => b.MustNot(Query.Terms(d => d.Field(_stackIdFieldName).Terms(excludedStackIds)))); + ctx.Filter &= new BoolQuery { MustNot = [new TermsQuery { Field = StackIdField, Terms = new TermsQueryField(excludedStackIds.Select(FieldValueHelper.ToFieldValue).ToList()) }] }; return Task.CompletedTask; } diff --git a/src/Exceptionless.Core/Repositories/Queries/Visitors/StackDateFixedQueryVisitor.cs b/src/Exceptionless.Core/Repositories/Queries/Visitors/StackDateFixedQueryVisitor.cs index 95909ba731..b092dd0c8b 100644 --- a/src/Exceptionless.Core/Repositories/Queries/Visitors/StackDateFixedQueryVisitor.cs +++ b/src/Exceptionless.Core/Repositories/Queries/Visitors/StackDateFixedQueryVisitor.cs @@ -1,7 +1,7 @@ +using Elastic.Clients.Elasticsearch.QueryDsl; using Foundatio.Parsers.ElasticQueries.Extensions; using Foundatio.Parsers.LuceneQueries.Nodes; using Foundatio.Parsers.LuceneQueries.Visitors; -using Nest; namespace Exceptionless.Core.Repositories.Queries; @@ -23,8 +23,12 @@ public override Task VisitAsync(TermNode node, IQueryVisitorContext if (!Boolean.TryParse(node.Term, out bool isFixed)) return Task.FromResult(node); - var query = new ExistsQuery { Field = _dateFixedFieldName }; - node.SetQuery(isFixed ? query : !query); + var existsQuery = new ExistsQuery { Field = _dateFixedFieldName }; + Query query = isFixed + ? existsQuery + : new BoolQuery { MustNot = new Query[] { existsQuery } }; + + node.SetQuery(query); return Task.FromResult(node); } diff --git a/src/Exceptionless.Core/Repositories/SavedViewRepository.cs b/src/Exceptionless.Core/Repositories/SavedViewRepository.cs index 694e008ccb..42ff830d63 100644 --- a/src/Exceptionless.Core/Repositories/SavedViewRepository.cs +++ b/src/Exceptionless.Core/Repositories/SavedViewRepository.cs @@ -3,7 +3,6 @@ using Exceptionless.Core.Validation; using Foundatio.Repositories; using Foundatio.Repositories.Models; -using Nest; namespace Exceptionless.Core.Repositories; @@ -19,7 +18,7 @@ public Task> GetByViewAsync(string organizationId, string return FindAsync(q => q .Organization(organizationId) .FieldEquals(e => e.ViewType, viewType) - .SortAscending(e => e.Name.Suffix("keyword")), options); + .SortAscending(e => e.Name), options); } public Task> GetByViewForUserAsync(string organizationId, string viewType, string userId, CommandOptionsDescriptor? options = null) @@ -27,7 +26,7 @@ public Task> GetByViewForUserAsync(string organizationId, return FindAsync(q => q .Organization(organizationId) .FieldEquals(e => e.ViewType, viewType) - .SortAscending(e => e.Name.Suffix("keyword")) + .SortAscending(e => e.Name) .FieldOr(g => g .FieldEmpty(e => e.UserId!) .FieldEquals(e => e.UserId!, userId)), options); @@ -37,7 +36,7 @@ public Task> GetByOrganizationForUserAsync(string organiz { return FindAsync(q => q .Organization(organizationId) - .SortAscending(e => e.Name.Suffix("keyword")) + .SortAscending(e => e.Name) .FieldOr(g => g .FieldEmpty(e => e.UserId!) .FieldEquals(e => e.UserId!, userId)), options); diff --git a/src/Exceptionless.Core/Repositories/StackRepository.cs b/src/Exceptionless.Core/Repositories/StackRepository.cs index d436ffa52a..29b0c3407e 100644 --- a/src/Exceptionless.Core/Repositories/StackRepository.cs +++ b/src/Exceptionless.Core/Repositories/StackRepository.cs @@ -7,7 +7,6 @@ using Foundatio.Repositories.Models; using Foundatio.Repositories.Options; using Microsoft.Extensions.Logging; -using Nest; namespace Exceptionless.Core.Repositories; @@ -25,14 +24,14 @@ public StackRepository(ExceptionlessElasticConfiguration configuration, MiniVali public Task> GetExpiredSnoozedStatuses(DateTime utcNow, CommandOptionsDescriptor? options = null) { - return FindAsync(q => q.ElasticFilter(Query.DateRange(d => d.Field(f => f.SnoozeUntilUtc).LessThanOrEquals(utcNow))), options); + return FindAsync(q => q.DateRange(null, utcNow, (Stack s) => s.SnoozeUntilUtc), options); } public Task> GetStacksForCleanupAsync(string organizationId, DateTime cutoff) { return FindAsync(q => q .Organization(organizationId) - .ElasticFilter(Query.DateRange(d => d.Field(f => f.LastOccurrence).LessThanOrEquals(cutoff))) + .DateRange(null, cutoff, (Stack s) => s.LastOccurrence) .FieldEquals(f => f.Status, StackStatus.Open) .FieldEmpty(f => f.References) .Include(f => f.Id, f => f.OrganizationId, f => f.ProjectId, f => f.SignatureHash) @@ -155,7 +154,7 @@ Instant parseDate(def dt) { public async Task GetStackBySignatureHashAsync(string projectId, string signatureHash) { string key = GetStackSignatureCacheKey(projectId, signatureHash); - var hit = await FindOneAsync(q => q.Project(projectId).ElasticFilter(Query.Term(s => s.SignatureHash, signatureHash)), o => o.Cache(key)); + var hit = await FindOneAsync(q => q.Project(projectId).FieldEquals(s => s.SignatureHash, signatureHash), o => o.Cache(key)); return hit?.Document; } @@ -194,7 +193,7 @@ protected override async Task AddDocumentsToCacheAsync(ICollection>(); foreach (var hit in findHits.Where(d => !String.IsNullOrEmpty(d.Document?.SignatureHash))) - cacheEntries.Add(GetStackSignatureCacheKey(hit.Document!), hit); + cacheEntries[GetStackSignatureCacheKey(hit.Document!)] = hit; if (cacheEntries.Count > 0) await AddDocumentsToCacheWithKeyAsync(cacheEntries, options.GetExpiresIn()); diff --git a/src/Exceptionless.Core/Repositories/TokenRepository.cs b/src/Exceptionless.Core/Repositories/TokenRepository.cs index ee6443862d..98db3d5e1f 100644 --- a/src/Exceptionless.Core/Repositories/TokenRepository.cs +++ b/src/Exceptionless.Core/Repositories/TokenRepository.cs @@ -4,7 +4,6 @@ using Exceptionless.Core.Validation; using Foundatio.Repositories; using Foundatio.Repositories.Models; -using Nest; using Token = Exceptionless.Core.Models.Token; namespace Exceptionless.Core.Repositories; @@ -19,36 +18,39 @@ public TokenRepository(ExceptionlessElasticConfiguration configuration, MiniVali public Task> GetByTypeAndUserIdAsync(TokenType type, string userId, CommandOptionsDescriptor? options = null) { - var filter = Query.Term(e => e.UserId, userId) && Query.Term(t => t.Type, type); - return FindAsync(q => q.ElasticFilter(filter).Sort(f => f.CreatedUtc), options); + return FindAsync(q => q.FieldEquals(t => t.UserId, userId).FieldEquals(t => t.Type, (int)type).Sort(f => f.CreatedUtc), options); } public Task> GetByTypeAndOrganizationIdAsync(TokenType type, string organizationId, CommandOptionsDescriptor? options = null) { return FindAsync(q => q .Organization(organizationId) - .ElasticFilter(Query.Term(t => t.Type, type)) + .FieldEquals(t => t.Type, (int)type) .Sort(f => f.CreatedUtc), options); } public Task> GetByTypeAndProjectIdAsync(TokenType type, string projectId, CommandOptionsDescriptor? options = null) { - var filter = ( - Query.Term(t => t.ProjectId, projectId) || Query.Term(t => t.DefaultProjectId, projectId) - ) && Query.Term(t => t.Type, type); - - return FindAsync(q => q.ElasticFilter(filter).Sort(f => f.CreatedUtc), options); + return FindAsync(q => q + .FieldOr(g => g + .FieldEquals(t => t.ProjectId, projectId) + .FieldEquals(t => t.DefaultProjectId, projectId)) + .FieldEquals(t => t.Type, (int)type) + .Sort(f => f.CreatedUtc), options); } public override Task> GetByProjectIdAsync(string projectId, CommandOptionsDescriptor? options = null) { - var filter = (Query.Term(t => t.ProjectId, projectId) || Query.Term(t => t.DefaultProjectId, projectId)); - return FindAsync(q => q.ElasticFilter(filter).Sort(f => f.CreatedUtc), options); + return FindAsync(q => q + .FieldOr(g => g + .FieldEquals(t => t.ProjectId, projectId) + .FieldEquals(t => t.DefaultProjectId, projectId)) + .Sort(f => f.CreatedUtc), options); } public Task RemoveAllByUserIdAsync(string userId, CommandOptionsDescriptor? options = null) { - return RemoveAllAsync(q => q.ElasticFilter(Query.Term(t => t.UserId, userId)), options); + return RemoveAllAsync(q => q.FieldEquals(t => t.UserId, userId), options); } protected override Task PublishChangeTypeMessageAsync(ChangeType changeType, Token? document, IDictionary? data = null, TimeSpan? delay = null) diff --git a/src/Exceptionless.Core/Repositories/UserRepository.cs b/src/Exceptionless.Core/Repositories/UserRepository.cs index d54ae2f5c0..77cb5296b9 100644 --- a/src/Exceptionless.Core/Repositories/UserRepository.cs +++ b/src/Exceptionless.Core/Repositories/UserRepository.cs @@ -4,7 +4,6 @@ using Foundatio.Repositories; using Foundatio.Repositories.Models; using Foundatio.Repositories.Options; -using Nest; using User = Exceptionless.Core.Models.User; namespace Exceptionless.Core.Repositories; @@ -24,7 +23,7 @@ public UserRepository(ExceptionlessElasticConfiguration configuration, MiniValid return null; emailAddress = emailAddress.Trim().ToLowerInvariant(); - var hit = await FindOneAsync(q => q.ElasticFilter(Query.Term(u => u.EmailAddress.Suffix("keyword"), emailAddress)), o => o.Cache(EmailCacheKey(emailAddress))); + var hit = await FindOneAsync(q => q.FieldEquals(u => u.EmailAddress, emailAddress), o => o.Cache(EmailCacheKey(emailAddress))); return hit?.Document; } @@ -33,7 +32,7 @@ public UserRepository(ExceptionlessElasticConfiguration configuration, MiniValid if (String.IsNullOrEmpty(token)) return null; - var hit = await FindOneAsync(q => q.ElasticFilter(Query.Term(u => u.PasswordResetToken, token))); + var hit = await FindOneAsync(q => q.FieldEquals(u => u.PasswordResetToken, token)); return hit?.Document; } @@ -43,8 +42,7 @@ public UserRepository(ExceptionlessElasticConfiguration configuration, MiniValid return null; provider = provider.ToLowerInvariant(); - var filter = Query.Term(u => u.OAuthAccounts.First().ProviderUserId, providerUserId); - var results = (await FindAsync(q => q.ElasticFilter(filter))).Documents; + var results = (await FindAsync(q => q.FieldEquals(u => u.OAuthAccounts.First().ProviderUserId, providerUserId))).Documents; return results.FirstOrDefault(u => u.OAuthAccounts.Any(o => o.Provider == provider)); } @@ -53,8 +51,7 @@ public UserRepository(ExceptionlessElasticConfiguration configuration, MiniValid if (String.IsNullOrEmpty(token)) return null; - var filter = Query.Term(u => u.VerifyEmailAddressToken, token); - var hit = await FindOneAsync(q => q.ElasticFilter(filter)); + var hit = await FindOneAsync(q => q.FieldEquals(u => u.VerifyEmailAddressToken, token)); return hit?.Document; } @@ -67,8 +64,7 @@ public Task> GetByOrganizationIdAsync(string organizationId, C if (commandOptions.ShouldUseCache()) throw new Exception("Caching of paged queries is not allowed"); - var filter = Query.Term(u => u.OrganizationIds, organizationId); - return FindAsync(q => q.ElasticFilter(filter).SortAscending(u => u.EmailAddress.Suffix("keyword")), o => commandOptions); + return FindAsync(q => q.FieldEquals(u => u.OrganizationIds, organizationId).SortAscending(u => u.EmailAddress), o => commandOptions); } protected override async Task AddDocumentsToCacheAsync(ICollection> findHits, ICommandOptions options, bool isDirtyRead) diff --git a/src/Exceptionless.Core/Repositories/WebHookRepository.cs b/src/Exceptionless.Core/Repositories/WebHookRepository.cs index 608e59caf4..b504fec2cf 100644 --- a/src/Exceptionless.Core/Repositories/WebHookRepository.cs +++ b/src/Exceptionless.Core/Repositories/WebHookRepository.cs @@ -5,7 +5,6 @@ using Foundatio.Repositories; using Foundatio.Repositories.Models; using Microsoft.Extensions.Logging; -using Nest; namespace Exceptionless.Core.Repositories; @@ -27,8 +26,14 @@ public Task> GetByOrganizationIdOrProjectIdAsync(string org ArgumentException.ThrowIfNullOrEmpty(organizationId); ArgumentException.ThrowIfNullOrEmpty(projectId); - var filter = (Query.Term(e => e.OrganizationId, organizationId) && !Query.Exists(e => e.Field(f => f.ProjectId))) || Query.Term(e => e.ProjectId, projectId); - return FindAsync(q => q.ElasticFilter(filter).Sort(f => f.CreatedUtc), o => o.Cache(PagedCacheKey(organizationId, projectId))); + // Match organization-level webhooks (organization matches, no project) OR project-specific webhooks + return FindAsync(q => q + .FieldOr(g => g + .FieldAnd(a => a + .FieldEquals(w => w.OrganizationId, organizationId) + .FieldEmpty(w => w.ProjectId)) + .FieldEquals(w => w.ProjectId, projectId)) + .Sort(f => f.CreatedUtc), o => o.Cache(PagedCacheKey(organizationId, projectId))); } public override Task> GetByProjectIdAsync(string projectId, CommandOptionsDescriptor? options = null) diff --git a/src/Exceptionless.Core/Serialization/DataObjectConverter.cs b/src/Exceptionless.Core/Serialization/DataObjectConverter.cs deleted file mode 100644 index 559c952253..0000000000 --- a/src/Exceptionless.Core/Serialization/DataObjectConverter.cs +++ /dev/null @@ -1,197 +0,0 @@ -using System.Collections.Concurrent; -using System.Reflection; -using Exceptionless.Core.Extensions; -using Exceptionless.Core.Models; -using Exceptionless.Core.Reflection; -using Foundatio.Repositories.Extensions; -using Microsoft.Extensions.Logging; -using Newtonsoft.Json; -using Newtonsoft.Json.Converters; -using Newtonsoft.Json.Linq; - -namespace Exceptionless.Serializer; - -public class DataObjectConverter : CustomCreationConverter where T : IData, new() -{ - private static readonly Type _type = typeof(T); - private static readonly ConcurrentDictionary _propertyAccessors = new(StringComparer.OrdinalIgnoreCase); - private readonly ConcurrentDictionary _dataTypeRegistry = new(StringComparer.OrdinalIgnoreCase); - private readonly ILogger _logger; - private readonly char[] _filteredChars = ['.', '-', '_']; - - public DataObjectConverter(ILogger logger, IEnumerable>? knownDataTypes = null) - { - _logger = logger; - - if (knownDataTypes is not null) - _dataTypeRegistry.AddRange(knownDataTypes); - - if (_propertyAccessors.Count != 0) - return; - - foreach (var prop in _type.GetProperties(BindingFlags.Instance | BindingFlags.FlattenHierarchy | BindingFlags.Public).Where(p => p.CanWrite)) - _propertyAccessors.TryAdd(prop.Name, LateBinder.GetPropertyAccessor(prop)); - } - - public void AddKnownDataType(string name, Type dataType) - { - _dataTypeRegistry.TryAdd(name, dataType); - } - - public override object ReadJson(JsonReader reader, Type objectType, object? existingValue, JsonSerializer serializer) - { - var target = Create(objectType); - var json = JObject.Load(reader); - - foreach (var p in json.Properties()) - { - string propertyName = p.Name.ToLowerFiltered(_filteredChars); - - if (propertyName == "data" && p.Value is JObject) - { - foreach (var dataProp in ((JObject)p.Value).Properties()) - AddDataEntry(serializer, dataProp, target); - - continue; - } - - var accessor = _propertyAccessors.TryGetValue(propertyName, out var value) ? value : null; - if (accessor is not null) - { - if (p.Value.Type == JTokenType.None || p.Value.Type == JTokenType.Undefined) - continue; - - if (p.Value.Type == JTokenType.Null) - { - accessor.SetValue(target, null); - continue; - } - - if (accessor.MemberType == typeof(DateTime)) - { - if (p.Value.Type == JTokenType.Date || p.Value.Type == JTokenType.String && p.Value.Value()!.Contains("+")) - { - accessor.SetValue(target, p.Value.ToObject(serializer).DateTime); - continue; - } - } - else if (accessor.MemberType == typeof(DateTime?)) - { - if (p.Value.Type == JTokenType.Date || p.Value.Type == JTokenType.String && p.Value.Value()!.Contains("+")) - { - var offset = p.Value.ToObject(serializer); - accessor.SetValue(target, offset?.DateTime); - continue; - } - } - - accessor.SetValue(target, p.Value.ToObject(accessor.MemberType, serializer)); - continue; - } - - AddDataEntry(serializer, p, target); - } - - return target; - } - - private void AddDataEntry(JsonSerializer serializer, JProperty p, T target) - { - if (target.Data is null) - target.Data = new DataDictionary(); - - string dataKey = GetDataKey(target.Data, p.Name); - string unknownTypeDataKey = GetDataKey(target.Data, p.Name, true); - - // when adding items to data, see if they are a known type and deserialize to the registered type - if (_dataTypeRegistry.TryGetValue(p.Name, out var dataType)) - { - try - { - if (p.Value is JValue && p.Value.Type == JTokenType.String) - { - string value = p.Value.ToString(); - if (value.IsJson()) - target.Data[dataKey] = serializer.Deserialize(new StringReader(value), dataType); - else - target.Data[dataType == value.GetType() ? dataKey : unknownTypeDataKey] = value; - } - else - { - target.Data[dataKey] = p.Value.ToObject(dataType, serializer); - } - - return; - } - catch (Exception) - { - _logger.LogInformation("Error deserializing known data type {Name}: {Value}", p.Name, p.Value.ToString()); - } - } - - // Add item to data as a JObject, JArray or native type. - if (p.Value is JObject) - { - target.Data[dataType is null || dataType == typeof(JObject) ? dataKey : unknownTypeDataKey] = p.Value.ToObject(); - } - else if (p.Value is JArray) - { - target.Data[dataType is null || dataType == typeof(JArray) ? dataKey : unknownTypeDataKey] = p.Value.ToObject(); - } - else if (p.Value is JValue jValue && jValue.Type != JTokenType.String) - { - object? value = jValue.Value; - target.Data[dataType is null || dataType == value?.GetType() ? dataKey : unknownTypeDataKey] = value; - } - else - { - string value = p.Value.ToString(); - var jsonType = value.GetJsonType(); - if (jsonType == JsonType.Object) - { - if (value.TryFromJson(out JObject? obj)) - target.Data[dataType is null || dataType == obj?.GetType() ? dataKey : unknownTypeDataKey] = obj; - else - target.Data[dataType is null || dataType == value.GetType() ? dataKey : unknownTypeDataKey] = value; - } - else if (jsonType == JsonType.Array) - { - if (value.TryFromJson(out JArray? obj)) - target.Data[dataType is null || dataType == obj?.GetType() ? dataKey : unknownTypeDataKey] = obj; - else - target.Data[dataType is null || dataType == value.GetType() ? dataKey : unknownTypeDataKey] = value; - } - else - { - target.Data[dataType is null || dataType == value.GetType() ? dataKey : unknownTypeDataKey] = value; - } - } - } - - private string GetDataKey(DataDictionary data, string dataKey, bool isUnknownType = false) - { - if (data.ContainsKey(dataKey) || isUnknownType) - dataKey = dataKey.StartsWith("@") ? "_" + dataKey : dataKey; - - int count = 1; - string key = dataKey; - while (data.ContainsKey(key) || (isUnknownType && _dataTypeRegistry.ContainsKey(key))) - key = dataKey + count++; - - return key; - } - - public override T Create(Type objectType) - { - return new T(); - } - - public override bool CanRead => true; - - public override bool CanWrite => false; - - public override bool CanConvert(Type objectType) - { - return objectType == _type; - } -} diff --git a/src/Exceptionless.Core/Serialization/DynamicTypeContractResolver.cs b/src/Exceptionless.Core/Serialization/DynamicTypeContractResolver.cs deleted file mode 100644 index cde0f17ccb..0000000000 --- a/src/Exceptionless.Core/Serialization/DynamicTypeContractResolver.cs +++ /dev/null @@ -1,38 +0,0 @@ -using System.Reflection; -using Foundatio.Repositories.Extensions; -using Newtonsoft.Json.Serialization; - -namespace Exceptionless.Serializer; - -public class DynamicTypeContractResolver : IContractResolver -{ - private readonly HashSet _assemblies = new(); - private readonly HashSet _types = new(); - - private readonly IContractResolver _defaultResolver; - private readonly IContractResolver _resolver; - - public DynamicTypeContractResolver(IContractResolver resolver, IContractResolver? defaultResolver = null) - { - _resolver = resolver ?? throw new ArgumentNullException(nameof(resolver)); - _defaultResolver = defaultResolver ?? new DefaultContractResolver(); - } - - public void UseDefaultResolverFor(params Assembly[] assemblies) - { - _assemblies.AddRange(assemblies); - } - - public void UseDefaultResolverFor(params Type[] types) - { - _types.AddRange(types); - } - - public JsonContract ResolveContract(Type type) - { - if (_types.Contains(type) || _assemblies.Contains(type.Assembly)) - return _defaultResolver.ResolveContract(type); - - return _resolver.ResolveContract(type); - } -} diff --git a/src/Exceptionless.Core/Serialization/ElasticConnectionSettingsAwareContractResolver.cs b/src/Exceptionless.Core/Serialization/ElasticConnectionSettingsAwareContractResolver.cs deleted file mode 100644 index 541befc681..0000000000 --- a/src/Exceptionless.Core/Serialization/ElasticConnectionSettingsAwareContractResolver.cs +++ /dev/null @@ -1,34 +0,0 @@ -using System.Reflection; -using Exceptionless.Core.Extensions; -using Nest; -using Nest.JsonNetSerializer; -using Newtonsoft.Json; -using Newtonsoft.Json.Serialization; - -namespace Exceptionless.Core.Serialization; - -public class ElasticConnectionSettingsAwareContractResolver : ConnectionSettingsAwareContractResolver -{ - public ElasticConnectionSettingsAwareContractResolver(IConnectionSettingsValues connectionSettings) : base(connectionSettings) { } - - protected override JsonProperty CreateProperty(MemberInfo member, MemberSerialization memberSerialization) - { - var property = base.CreateProperty(member, memberSerialization); - - var shouldSerialize = property.ShouldSerialize; - property.ShouldSerialize = obj => (shouldSerialize is null || shouldSerialize(obj)) && !property.IsValueEmptyCollection(obj); - return property; - } - - protected override JsonDictionaryContract CreateDictionaryContract(Type objectType) - { - var contract = base.CreateDictionaryContract(objectType); - contract.DictionaryKeyResolver = propertyName => propertyName; - return contract; - } - - protected override string ResolvePropertyName(string propertyName) - { - return propertyName.ToLowerUnderscoredWords(); - } -} diff --git a/src/Exceptionless.Core/Serialization/ElasticJsonNetSerializer.cs b/src/Exceptionless.Core/Serialization/ElasticJsonNetSerializer.cs deleted file mode 100644 index 403a2329e8..0000000000 --- a/src/Exceptionless.Core/Serialization/ElasticJsonNetSerializer.cs +++ /dev/null @@ -1,41 +0,0 @@ -using Elasticsearch.Net; -using Nest; -using Nest.JsonNetSerializer; -using Newtonsoft.Json; - -namespace Exceptionless.Core.Serialization; - -public class ElasticJsonNetSerializer : JsonNetSerializer -{ - public ElasticJsonNetSerializer( - IElasticsearchSerializer builtinSerializer, - IConnectionSettingsValues connectionSettings, - JsonSerializerSettings serializerSettings - ) : base( - builtinSerializer, - connectionSettings, - () => CreateJsonSerializerSettings(serializerSettings), - contractJsonConverters: serializerSettings.Converters.ToList() - ) - { - } - - private static JsonSerializerSettings CreateJsonSerializerSettings(JsonSerializerSettings serializerSettings) - { - return new JsonSerializerSettings - { - DateParseHandling = serializerSettings.DateParseHandling, - DefaultValueHandling = serializerSettings.DefaultValueHandling, - MissingMemberHandling = serializerSettings.MissingMemberHandling, - NullValueHandling = serializerSettings.NullValueHandling - }; - } - - protected override ConnectionSettingsAwareContractResolver CreateContractResolver() - { - // TODO: Verify we don't need to use the DynamicTypeContractResolver. - var resolver = new ElasticConnectionSettingsAwareContractResolver(ConnectionSettings); - ModifyContractResolver(resolver); - return resolver; - } -} diff --git a/src/Exceptionless.Core/Serialization/EmptyCollectionModifier.cs b/src/Exceptionless.Core/Serialization/EmptyCollectionModifier.cs new file mode 100644 index 0000000000..f85623cbe4 --- /dev/null +++ b/src/Exceptionless.Core/Serialization/EmptyCollectionModifier.cs @@ -0,0 +1,85 @@ +using System.Collections; +using System.Reflection; +using System.Text.Json.Serialization.Metadata; + +namespace Exceptionless.Core.Serialization; + +/// +/// A type info modifier that skips empty collections during serialization to match Newtonsoft's behavior. +/// +public static class EmptyCollectionModifier +{ + /// + /// Modifies JSON type info to skip empty collections/dictionaries during serialization. + /// + public static void SkipEmptyCollections(JsonTypeInfo typeInfo) + { + foreach (var property in typeInfo.Properties) + { + // For properties typed as IEnumerable (but not string), check at compile time + if (typeof(IEnumerable).IsAssignableFrom(property.PropertyType) && property.PropertyType != typeof(string)) + { + // Pre-resolve Count accessor at setup time to avoid reflection during serialization. + // Handles types like HashSet that implement ICollection but not non-generic ICollection. + var countAccessor = GetCountAccessor(property.PropertyType); + var originalShouldSerialize = property.ShouldSerialize; + + property.ShouldSerialize = (obj, value) => + { + if (originalShouldSerialize is not null && !originalShouldSerialize(obj, value)) + return false; + + return !IsEmptyCollection(value, countAccessor); + }; + } + // For object-typed properties, check the runtime value + else if (property.PropertyType == typeof(object)) + { + var originalShouldSerialize = property.ShouldSerialize; + property.ShouldSerialize = (obj, value) => + { + // First check original condition if any + if (originalShouldSerialize is not null && !originalShouldSerialize(obj, value)) + return false; + + // Then check if runtime value is an empty collection + return !IsEmptyCollection(value, null); + }; + } + } + } + + private static bool IsEmptyCollection(object? value, PropertyInfo? countAccessor) + { + return value switch + { + // Setting ShouldSerialize on a property can bypass DefaultIgnoreCondition.WhenWritingNull + // in some .NET versions, so we must explicitly handle null here. + null => true, + string => false, // strings are IEnumerable but should not be treated as collections + ICollection { Count: 0 } => true, // List, Dictionary, arrays + _ => countAccessor is not null && (int)countAccessor.GetValue(value)! == 0 + }; + } + + /// + /// Resolves a Count property accessor for types that implement ICollection{T} but not ICollection. + /// Called once at type-info setup time; the PropertyInfo is reused for all serializations. + /// + private static PropertyInfo? GetCountAccessor(Type propertyType) + { + // If it already implements non-generic ICollection, the pattern match handles it fast. + if (typeof(ICollection).IsAssignableFrom(propertyType)) + return null; + + // Check if it implements ICollection (e.g., HashSet) + foreach (var iface in propertyType.GetInterfaces()) + { + if (iface.IsGenericType && iface.GetGenericTypeDefinition() == typeof(ICollection<>)) + return iface.GetProperty("Count"); + } + + // Fallback: any IEnumerable with a public Count property (custom collections) + return propertyType.GetProperty("Count"); + } +} diff --git a/src/Exceptionless.Core/Serialization/ExceptionlessNamingStrategy.cs b/src/Exceptionless.Core/Serialization/ExceptionlessNamingStrategy.cs deleted file mode 100644 index d677cf08d8..0000000000 --- a/src/Exceptionless.Core/Serialization/ExceptionlessNamingStrategy.cs +++ /dev/null @@ -1,12 +0,0 @@ -using Exceptionless.Core.Extensions; -using Newtonsoft.Json.Serialization; - -namespace Exceptionless.Core.Serialization; - -public class ExceptionlessNamingStrategy : SnakeCaseNamingStrategy -{ - protected override string ResolvePropertyName(string name) - { - return name.ToLowerUnderscoredWords(); - } -} diff --git a/src/Exceptionless.Core/Serialization/JsonElementConverter.cs b/src/Exceptionless.Core/Serialization/JsonElementConverter.cs new file mode 100644 index 0000000000..584ca488cb --- /dev/null +++ b/src/Exceptionless.Core/Serialization/JsonElementConverter.cs @@ -0,0 +1,47 @@ +using System.Text.Json; + +namespace Exceptionless.Core.Serialization; + +public static class JsonElementConverter +{ + public static object? Convert(JsonElement element) + { + return element.ValueKind switch + { + JsonValueKind.String => ConvertString(element), + JsonValueKind.Number => JsonNumberInference.Convert(element), + JsonValueKind.True => true, + JsonValueKind.False => false, + JsonValueKind.Null => null, + JsonValueKind.Array => element.EnumerateArray() + .Select(Convert) + .ToList(), + JsonValueKind.Object => ConvertObject(element), + _ => element.GetRawText() + }; + } + + private static Dictionary ConvertObject(JsonElement element) + { + var dict = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (JsonProperty prop in element.EnumerateObject()) + dict[prop.Name] = Convert(prop.Value); + + return dict; + } + + private static object? ConvertString(JsonElement element) + { + string? raw = element.GetRawText(); + if (raw is not null && raw.Contains('T')) + { + if (element.TryGetDateTimeOffset(out DateTimeOffset dateTimeOffset)) + return dateTimeOffset; + + if (element.TryGetDateTime(out DateTime dt)) + return dt; + } + + return element.GetString(); + } +} diff --git a/src/Exceptionless.Core/Serialization/JsonNumberInference.cs b/src/Exceptionless.Core/Serialization/JsonNumberInference.cs new file mode 100644 index 0000000000..fc0c100b7a --- /dev/null +++ b/src/Exceptionless.Core/Serialization/JsonNumberInference.cs @@ -0,0 +1,141 @@ +using System.Buffers; +using System.Text.Json; + +namespace Exceptionless.Core.Serialization; + +internal static class JsonNumberInference +{ + public static object Read(ref Utf8JsonReader reader, bool preferInt64) + { + ReadOnlySpan rawValue = reader.HasValueSequence + ? reader.ValueSequence.ToArray() + : reader.ValueSpan; + + if (HasDecimalOrExponent(rawValue)) + { + if (preferInt64) + return reader.GetDouble(); + + if (reader.TryGetDecimal(out decimal d)) + { + if (ShouldPreserveDecimalZero(rawValue, d)) + return d; + + return reader.GetDouble(); + } + + return reader.GetDouble(); + } + + if (preferInt64) + { + if (reader.TryGetInt64(out long l)) + return l; + + return reader.GetDouble(); + } + + if (reader.TryGetInt32(out int i)) + return i; + + if (reader.TryGetInt64(out long longValue)) + return longValue; + + if (reader.TryGetDecimal(out decimal decimalValue)) + return decimalValue; + + return reader.GetDouble(); + } + + public static object Convert(JsonElement element) + { + string rawText = element.GetRawText(); + if (HasDecimalOrExponent(rawText.AsSpan())) + { + if (element.TryGetDecimal(out decimal d)) + { + if (ShouldPreserveDecimalZero(rawText.AsSpan(), d)) + return d; + + return element.GetDouble(); + } + + return element.GetDouble(); + } + + if (element.TryGetInt32(out int i)) + return i; + + if (element.TryGetInt64(out long l)) + return l; + + if (element.TryGetDecimal(out decimal dec)) + return dec; + + return element.GetDouble(); + } + + private static bool HasDecimalOrExponent(ReadOnlySpan rawValue) + { + return rawValue.Contains((byte)'.') || rawValue.Contains((byte)'e') || rawValue.Contains((byte)'E'); + } + + private static bool HasDecimalOrExponent(ReadOnlySpan rawValue) + { + return rawValue.Contains('.') || rawValue.Contains('e') || rawValue.Contains('E'); + } + + private static bool ShouldPreserveDecimalZero(ReadOnlySpan rawValue, decimal value) + { + if (value != 0m) + return true; + + return RepresentsZero(rawValue) && !IsNegative(rawValue); + } + + private static bool ShouldPreserveDecimalZero(ReadOnlySpan rawValue, decimal value) + { + if (value != 0m) + return true; + + return RepresentsZero(rawValue) && !IsNegative(rawValue); + } + + private static bool IsNegative(ReadOnlySpan rawValue) + { + return rawValue.Length > 0 && rawValue[0] == (byte)'-'; + } + + private static bool IsNegative(ReadOnlySpan rawValue) + { + return rawValue.Length > 0 && rawValue[0] == '-'; + } + + private static bool RepresentsZero(ReadOnlySpan rawValue) + { + foreach (byte c in rawValue) + { + if (c is (byte)'e' or (byte)'E') + break; + + if (c is >= (byte)'1' and <= (byte)'9') + return false; + } + + return true; + } + + private static bool RepresentsZero(ReadOnlySpan rawValue) + { + foreach (char c in rawValue) + { + if (c is 'e' or 'E') + break; + + if (c is >= '1' and <= '9') + return false; + } + + return true; + } +} diff --git a/src/Exceptionless.Core/Serialization/JsonSerializerOptionsExtensions.cs b/src/Exceptionless.Core/Serialization/JsonSerializerOptionsExtensions.cs index 7a05de2ae7..cd27fb785e 100644 --- a/src/Exceptionless.Core/Serialization/JsonSerializerOptionsExtensions.cs +++ b/src/Exceptionless.Core/Serialization/JsonSerializerOptionsExtensions.cs @@ -1,5 +1,8 @@ +using System.Text.Encodings.Web; using System.Text.Json; using System.Text.Json.Serialization; +using System.Text.Json.Serialization.Metadata; +using System.Text.Unicode; namespace Exceptionless.Core.Serialization; @@ -9,24 +12,65 @@ namespace Exceptionless.Core.Serialization; public static class JsonSerializerOptionsExtensions { /// - /// Configures with Exceptionless conventions: - /// snake_case property naming, null value handling, and dynamic object support. + /// Configures with Exceptionless app conventions: + /// STJ snake_case property naming, null value handling, and dynamic object support. /// + /// + /// These defaults intentionally differ from Foundatio's repository defaults. API and storage + /// serialization omit nulls and empty collections by default, enforce nullable annotations for + /// typed app models, and use Exceptionless' dynamic object inference. Elasticsearch source + /// serialization layers repository defaults first, then applies the app-specific compatibility + /// overrides it needs. + /// /// The options to configure. /// The configured options for chaining. public static JsonSerializerOptions ConfigureExceptionlessDefaults(this JsonSerializerOptions options) + { + return ConfigureExceptionlessDefaults(options, skipEmptyCollections: true); + } + + /// + /// Configures API-specific response serialization on top of the shared naming and converter defaults. + /// + public static JsonSerializerOptions ConfigureExceptionlessApiDefaults(this JsonSerializerOptions options) + { + ConfigureExceptionlessDefaults(options, skipEmptyCollections: false); + return options; + } + + private static JsonSerializerOptions ConfigureExceptionlessDefaults(JsonSerializerOptions options, bool skipEmptyCollections) { options.DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull; - options.PropertyNamingPolicy = LowerCaseUnderscoreNamingPolicy.Instance; + options.PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower; + options.PropertyNameCaseInsensitive = true; + + // Allow non-ASCII Unicode (Chinese, Japanese, emoji, etc.) to pass through + // unescaped for readability. HTML-sensitive characters (<, >, &, ') are still + // escaped to their \uXXXX forms (e.g., & → \u0026); do not relax these escapes. + var encoderSettings = new TextEncoderSettings(UnicodeRanges.All); + encoderSettings.ForbidCharacter('<'); + encoderSettings.ForbidCharacter('>'); + encoderSettings.ForbidCharacter('&'); + encoderSettings.ForbidCharacter('\''); + encoderSettings.ForbidCharacter('"'); + options.Encoder = JavaScriptEncoder.Create(encoderSettings); + options.Converters.Add(new ObjectToInferredTypesConverter()); - // Ensures tuples and records are serialized with their field names instead of "Item1", "Item2", etc. + // Required for public-field value types that are intentionally serialized through + // the configured serializer, including ValueTuple cache keys and field-only structs. options.IncludeFields = true; - // Enforces C# nullable annotations (string vs string?) during serialization/deserialization. - // If you see "cannot be null" errors, fix the model's nullability annotation or the data. + // Enforces C# nullable annotations (string vs string?) for typed app models so bad + // payload/model contracts fail close to the boundary instead of silently storing nulls. + // Elasticsearch opts out below because historical documents may not match annotations. options.RespectNullableAnnotations = true; + var resolver = new DefaultJsonTypeInfoResolver(); + if (skipEmptyCollections) + resolver.Modifiers.Add(EmptyCollectionModifier.SkipEmptyCollections); + + options.TypeInfoResolver = resolver; return options; } } diff --git a/src/Exceptionless.Core/Serialization/LowerCaseUnderscoreNamingPolicy.cs b/src/Exceptionless.Core/Serialization/LowerCaseUnderscoreNamingPolicy.cs deleted file mode 100644 index 77952a322e..0000000000 --- a/src/Exceptionless.Core/Serialization/LowerCaseUnderscoreNamingPolicy.cs +++ /dev/null @@ -1,28 +0,0 @@ -using System.Text.Json; -using Exceptionless.Core.Extensions; - -namespace Exceptionless.Core.Serialization; - -/// -/// A JSON naming policy that converts PascalCase to lower_case_underscore format. -/// This uses the existing ToLowerUnderscoredWords extension method to maintain -/// API compatibility with legacy Newtonsoft.Json serialization. -/// -/// Note: This implementation treats each uppercase letter individually, so: -/// - "OSName" becomes "o_s_name" (not "os_name") -/// - "EnableSSL" becomes "enable_s_s_l" (not "enable_ssl") -/// - "BaseURL" becomes "base_u_r_l" (not "base_url") -/// - "PropertyName" becomes "property_name" -/// -/// This matches the legacy behavior. See https://github.com/exceptionless/Exceptionless.Net/issues/2 -/// for discussion on future improvements. -/// -public sealed class LowerCaseUnderscoreNamingPolicy : JsonNamingPolicy -{ - public static LowerCaseUnderscoreNamingPolicy Instance { get; } = new(); - - public override string ConvertName(string name) - { - return name.ToLowerUnderscoredWords(); - } -} diff --git a/src/Exceptionless.Core/Serialization/LowerCaseUnderscorePropertyNamesContractResolver.cs b/src/Exceptionless.Core/Serialization/LowerCaseUnderscorePropertyNamesContractResolver.cs deleted file mode 100644 index 7fe758b24d..0000000000 --- a/src/Exceptionless.Core/Serialization/LowerCaseUnderscorePropertyNamesContractResolver.cs +++ /dev/null @@ -1,30 +0,0 @@ -using System.Reflection; -using Exceptionless.Core.Extensions; -using Newtonsoft.Json; -using Newtonsoft.Json.Serialization; - -namespace Exceptionless.Core.Serialization; - -public class LowerCaseUnderscorePropertyNamesContractResolver : DefaultContractResolver -{ - protected override JsonProperty CreateProperty(MemberInfo member, MemberSerialization memberSerialization) - { - var property = base.CreateProperty(member, memberSerialization); - - var shouldSerialize = property.ShouldSerialize; - property.ShouldSerialize = obj => (shouldSerialize is null || shouldSerialize(obj)) && !property.IsValueEmptyCollection(obj); - return property; - } - - protected override JsonDictionaryContract CreateDictionaryContract(Type objectType) - { - var contract = base.CreateDictionaryContract(objectType); - contract.DictionaryKeyResolver = propertyName => propertyName; - return contract; - } - - protected override string ResolvePropertyName(string propertyName) - { - return propertyName.ToLowerUnderscoredWords(); - } -} diff --git a/src/Exceptionless.Core/Serialization/ObjectToInferredTypesConverter.cs b/src/Exceptionless.Core/Serialization/ObjectToInferredTypesConverter.cs index 8d5dd5841f..9165b16eea 100644 --- a/src/Exceptionless.Core/Serialization/ObjectToInferredTypesConverter.cs +++ b/src/Exceptionless.Core/Serialization/ObjectToInferredTypesConverter.cs @@ -1,7 +1,7 @@ +using System.Buffers; using System.Text.Json; using System.Text.Json.Serialization; using Exceptionless.Core.Models; -using Newtonsoft.Json.Linq; namespace Exceptionless.Core.Serialization; @@ -17,7 +17,7 @@ namespace Exceptionless.Core.Serialization; /// /// /// true/false → -/// Numbers → (if fits) or +/// Numbers → (if fits), , or ; with preferInt64, always for integers and for floats /// Strings with ISO 8601 date format → /// Other strings → /// null → null @@ -41,8 +41,32 @@ namespace Exceptionless.Core.Serialization; /// /// /// +/// +/// This converter is app-specific and NOT interchangeable with Foundatio.Repositories' +/// ObjectToInferredTypesConverter. Key differences: preferInt64 mode for ES compatibility, +/// aggressive DateTimeOffset detection from strings, int→long→decimal number inference. +/// public sealed class ObjectToInferredTypesConverter : JsonConverter { + private readonly bool _preferInt64; + + /// + /// Initializes a new instance with default settings (integers that fit Int32 are returned as ). + /// + public ObjectToInferredTypesConverter() : this(preferInt64: false) { } + + /// + /// Initializes a new instance with configurable integer handling. + /// + /// + /// When true, all integers are returned as to match JSON.NET behavior. + /// Used by the Elasticsearch serializer to maintain compatibility with DataObjectConverter. + /// + public ObjectToInferredTypesConverter(bool preferInt64) + { + _preferInt64 = preferInt64; + } + /// public override object? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { @@ -50,7 +74,7 @@ public sealed class ObjectToInferredTypesConverter : JsonConverter { JsonTokenType.True => true, JsonTokenType.False => false, - JsonTokenType.Number => ReadNumber(ref reader), + JsonTokenType.Number => JsonNumberInference.Read(ref reader, _preferInt64), JsonTokenType.String => ReadString(ref reader), JsonTokenType.Null => null, JsonTokenType.StartObject => ReadObject(ref reader, options), @@ -75,51 +99,37 @@ public override void Write(Utf8JsonWriter writer, object? value, JsonSerializerO return; } - // Handle Newtonsoft JToken types (stored in DataDictionary by DataObjectConverter - // when reading from Elasticsearch via NEST). Without this, STJ enumerates JToken's - // IEnumerable interface, producing nested empty arrays instead of proper JSON. - if (value is JToken jToken) - { - using var doc = JsonDocument.Parse(jToken.ToString(Newtonsoft.Json.Formatting.None)); - doc.RootElement.WriteTo(writer); - return; - } - // Serialize using the runtime type to get proper converter handling JsonSerializer.Serialize(writer, value, value.GetType(), options); } - /// - /// Reads a JSON number, preferring for integers and for decimals. - /// - private static object ReadNumber(ref Utf8JsonReader reader) - { - // Try smallest to largest integer types first for optimal boxing - if (reader.TryGetInt32(out int i)) - return i; - - if (reader.TryGetInt64(out long l)) - return l; - - // Try decimal for precise values (e.g., financial data) before double - if (reader.TryGetDecimal(out decimal d)) - return d; - - // Fall back to double for floating-point - return reader.GetDouble(); - } - /// /// Reads a JSON string, attempting to parse as for ISO 8601 dates. /// + /// + /// Only parses strings that contain the ISO 8601 time separator 'T'. + /// Date-only strings like "2026-01-15" are preserved as strings to match the legacy + /// Newtonsoft behavior (which used DateParseHandling.None for the Data dictionary). + /// private static object? ReadString(ref Utf8JsonReader reader) { - // Attempt DateTimeOffset parsing for ISO 8601 formatted strings - if (reader.TryGetDateTimeOffset(out DateTimeOffset dateTimeOffset)) - return dateTimeOffset; + // Check raw text for a time separator before attempting date parsing. + // Date-only strings ("2026-01-15") should stay as strings — they may not represent + // dates in user data. Only parse strings that have an explicit time component. + ReadOnlySpan rawValue = reader.HasValueSequence + ? reader.ValueSequence.ToArray() + : reader.ValueSpan; - if (reader.TryGetDateTime(out var dt)) - return dt; + bool hasTimeSeparator = rawValue.Contains((byte)'T'); + + if (hasTimeSeparator) + { + if (reader.TryGetDateTimeOffset(out DateTimeOffset dateTimeOffset)) + return dateTimeOffset; + + if (reader.TryGetDateTime(out var dt)) + return dt; + } return reader.GetString(); } @@ -131,7 +141,7 @@ private static object ReadNumber(ref Utf8JsonReader reader) /// Uses for property name matching, /// consistent with behavior. /// - private static Dictionary ReadObject(ref Utf8JsonReader reader, JsonSerializerOptions options) + private Dictionary ReadObject(ref Utf8JsonReader reader, JsonSerializerOptions options) { var dictionary = new Dictionary(StringComparer.OrdinalIgnoreCase); @@ -157,7 +167,7 @@ private static object ReadNumber(ref Utf8JsonReader reader) /// /// Recursively reads a JSON array into a of objects. /// - private static List ReadArray(ref Utf8JsonReader reader, JsonSerializerOptions options) + private List ReadArray(ref Utf8JsonReader reader, JsonSerializerOptions options) { var list = new List(); @@ -175,13 +185,13 @@ private static object ReadNumber(ref Utf8JsonReader reader) /// /// Reads a single JSON value of any type, dispatching to the appropriate reader method. /// - private static object? ReadValue(ref Utf8JsonReader reader, JsonSerializerOptions options) + private object? ReadValue(ref Utf8JsonReader reader, JsonSerializerOptions options) { return reader.TokenType switch { JsonTokenType.True => true, JsonTokenType.False => false, - JsonTokenType.Number => ReadNumber(ref reader), + JsonTokenType.Number => JsonNumberInference.Read(ref reader, _preferInt64), JsonTokenType.String => ReadString(ref reader), JsonTokenType.Null => null, JsonTokenType.StartObject => ReadObject(ref reader, options), @@ -189,4 +199,5 @@ private static object ReadNumber(ref Utf8JsonReader reader) _ => JsonDocument.ParseValue(ref reader).RootElement.Clone() }; } + } diff --git a/src/Exceptionless.Core/Services/SlackService.cs b/src/Exceptionless.Core/Services/SlackService.cs index 2154f284c9..7de5275616 100644 --- a/src/Exceptionless.Core/Services/SlackService.cs +++ b/src/Exceptionless.Core/Services/SlackService.cs @@ -14,7 +14,7 @@ public class SlackService private readonly HttpClient _client = new(); private readonly IQueue _webHookNotificationQueue; private readonly FormattingPluginManager _pluginManager; - private readonly ISerializer _serializer; + private readonly ITextSerializer _serializer; private readonly AppOptions _appOptions; private readonly ILogger _logger; @@ -117,7 +117,7 @@ public Task SendMessageAsync(string organizationId, string projectId, string url public async Task SendEventNoticeAsync(PersistentEvent ev, Project project, bool isNew, bool isRegression) { - var token = project.GetSlackToken(); + var token = project.GetSlackToken(_serializer, _logger); if (token?.IncomingWebhook?.Url is null) return false; diff --git a/src/Exceptionless.Core/Utility/ErrorSignature.cs b/src/Exceptionless.Core/Utility/ErrorSignature.cs index 9aa7d3c9f1..2762922fb2 100644 --- a/src/Exceptionless.Core/Utility/ErrorSignature.cs +++ b/src/Exceptionless.Core/Utility/ErrorSignature.cs @@ -1,8 +1,8 @@ using System.Text; -using System.Text.Json; using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Models.Data; +using Foundatio.Serializer; namespace Exceptionless.Core.Utility; @@ -10,14 +10,14 @@ public class ErrorSignature { private readonly HashSet _userNamespaces; private readonly HashSet _userCommonMethods; - private readonly JsonSerializerOptions _jsonOptions; + private readonly ITextSerializer _serializer; private static readonly string[] _defaultNonUserNamespaces = ["System", "Microsoft"]; // TODO: Add support for user public key token on signed assemblies - public ErrorSignature(Error error, JsonSerializerOptions jsonOptions, IEnumerable? userNamespaces = null, IEnumerable? userCommonMethods = null, bool emptyNamespaceIsUserMethod = true, bool shouldFlagSignatureTarget = true) + public ErrorSignature(Error error, ITextSerializer serializer, IEnumerable? userNamespaces = null, IEnumerable? userCommonMethods = null, bool emptyNamespaceIsUserMethod = true, bool shouldFlagSignatureTarget = true) { Error = error ?? throw new ArgumentNullException(nameof(error)); - _jsonOptions = jsonOptions ?? throw new ArgumentNullException(nameof(jsonOptions)); + _serializer = serializer ?? throw new ArgumentNullException(nameof(serializer)); _userNamespaces = userNamespaces is null ? [] @@ -180,7 +180,7 @@ private void AddSpecialCaseDetails(InnerError error) if (!error.Data.ContainsKey(Error.KnownDataKeys.ExtraProperties)) return; - var extraProperties = error.Data.GetValue>(Error.KnownDataKeys.ExtraProperties, _jsonOptions); + var extraProperties = error.Data.GetValue>(Error.KnownDataKeys.ExtraProperties, _serializer); if (extraProperties is null) { error.Data.Remove(Error.KnownDataKeys.ExtraProperties); diff --git a/src/Exceptionless.Core/Utility/ExtensibleObject.cs b/src/Exceptionless.Core/Utility/ExtensibleObject.cs index 6ac04bddf7..26f398c877 100644 --- a/src/Exceptionless.Core/Utility/ExtensibleObject.cs +++ b/src/Exceptionless.Core/Utility/ExtensibleObject.cs @@ -1,7 +1,6 @@ using System.ComponentModel; +using System.Text.Json; using Exceptionless.Core.Extensions; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; namespace Exceptionless.Core.Utility; @@ -9,7 +8,7 @@ public interface IExtensibleObject { void SetProperty(string name, T value); - T? GetProperty(string name); + T? GetProperty(string name, JsonSerializerOptions? options = null); object? GetProperty(string name); @@ -22,7 +21,6 @@ public interface IExtensibleObject public class ExtensibleObject : INotifyPropertyChanged, IExtensibleObject { - [JsonProperty] private readonly Dictionary _extendedData = new(); public void SetProperty(string name, T value) @@ -35,7 +33,7 @@ public void SetProperty(string name, T value) NotifyPropertyChanged(name); } - public T? GetProperty(string name) + public T? GetProperty(string name, JsonSerializerOptions? options = null) { object? value = GetProperty(name); if (value is null) @@ -44,8 +42,18 @@ public void SetProperty(string name, T value) if (value is T tValue) return tValue; - if (value is JContainer container) - return container.ToObject(); + // Handle JsonElement from STJ deserialization + if (value is JsonElement jsonElement) + { + try + { + return jsonElement.Deserialize(options); + } + catch (JsonException) + { + // Fall through to ToType conversion + } + } return value.ToType(); } diff --git a/src/Exceptionless.Core/Utility/TypeHelper.cs b/src/Exceptionless.Core/Utility/TypeHelper.cs index de21067616..bb429fa08d 100644 --- a/src/Exceptionless.Core/Utility/TypeHelper.cs +++ b/src/Exceptionless.Core/Utility/TypeHelper.cs @@ -1,7 +1,7 @@ using System.Diagnostics; using System.Globalization; using System.Reflection; -using Newtonsoft.Json.Linq; +using System.Text.Json; namespace Exceptionless.Core.Helpers; @@ -52,8 +52,9 @@ public static bool AreSameValue(object a, object b) catch { } } - if (a is JToken && b is JToken) - return String.Equals(a.ToString(), b.ToString()); + // Handle JsonElement comparison semantically + if (a is JsonElement jsonA && b is JsonElement jsonB) + return JsonElement.DeepEquals(jsonA, jsonB); if (a != b && !a.Equals(b)) return false; diff --git a/src/Exceptionless.Insulation/Bootstrapper.cs b/src/Exceptionless.Insulation/Bootstrapper.cs index 6174ca2cc3..5ce4f4f7b3 100644 --- a/src/Exceptionless.Insulation/Bootstrapper.cs +++ b/src/Exceptionless.Insulation/Bootstrapper.cs @@ -23,7 +23,6 @@ using Foundatio.Serializer; using Foundatio.Storage; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Logging; using Serilog.Sinks.Exceptionless; using StackExchange.Redis; diff --git a/src/Exceptionless.Insulation/HealthChecks/ElasticsearchHealthCheck.cs b/src/Exceptionless.Insulation/HealthChecks/ElasticsearchHealthCheck.cs index 4d191de743..b44676e0be 100644 --- a/src/Exceptionless.Insulation/HealthChecks/ElasticsearchHealthCheck.cs +++ b/src/Exceptionless.Insulation/HealthChecks/ElasticsearchHealthCheck.cs @@ -1,9 +1,7 @@ using System.Diagnostics; -using Elasticsearch.Net; using Exceptionless.Core.Repositories.Configuration; using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Logging; -using Nest; namespace Exceptionless.Insulation.HealthChecks; @@ -24,14 +22,8 @@ public async Task CheckHealthAsync(HealthCheckContext context try { - var pingResult = await _config.Client.LowLevel.PingAsync(ctx: cancellationToken, requestParameters: new PingRequestParameters - { - RequestConfiguration = new RequestConfiguration - { - RequestTimeout = TimeSpan.FromSeconds(60) // 60 seconds is default for NEST - } - }); - bool isSuccess = pingResult.ApiCall.HttpStatusCode == 200; + var pingResult = await _config.Client.PingAsync(cancellationToken); + bool isSuccess = pingResult.IsValidResponse; return isSuccess ? HealthCheckResult.Healthy() : new HealthCheckResult(context.Registration.FailureStatus); } diff --git a/src/Exceptionless.Web/Bootstrapper.cs b/src/Exceptionless.Web/Bootstrapper.cs index f83edd3615..e330addfdf 100644 --- a/src/Exceptionless.Web/Bootstrapper.cs +++ b/src/Exceptionless.Web/Bootstrapper.cs @@ -1,7 +1,5 @@ using Exceptionless.Core; -using Exceptionless.Core.Extensions; using Exceptionless.Core.Jobs.WorkItemHandlers; -using Exceptionless.Core.Queues.Models; using Exceptionless.Web.Hubs; using Exceptionless.Web.Mapping; using Foundatio.Extensions.Hosting.Startup; diff --git a/src/Exceptionless.Web/ClientApp/e2e/index.test.ts b/src/Exceptionless.Web/ClientApp/e2e/index.test.ts index b7abc1c705..7d109a8610 100644 --- a/src/Exceptionless.Web/ClientApp/e2e/index.test.ts +++ b/src/Exceptionless.Web/ClientApp/e2e/index.test.ts @@ -3,5 +3,5 @@ import { expect, test } from '@playwright/test'; test('default route should redirect to login page when unauthorized', async ({ page }) => { await page.goto('/next'); await page.waitForURL('/next/login'); - await expect(page.getByRole('button', { name: 'Login' })).toBeVisible(); + await expect(page.getByRole('button', { exact: true, name: 'Login' })).toBeVisible(); }); diff --git a/src/Exceptionless.Web/Controllers/AdminController.cs b/src/Exceptionless.Web/Controllers/AdminController.cs index b8f2d6b009..a27988603d 100644 --- a/src/Exceptionless.Web/Controllers/AdminController.cs +++ b/src/Exceptionless.Web/Controllers/AdminController.cs @@ -298,37 +298,34 @@ await _workItemQueue.EnqueueAsync(new UpdateProjectNotificationSettingsWorkItem public async Task> GetElasticsearchInfoAsync() { var client = _configuration.Client; - var healthTask = client.Cluster.HealthAsync(); + var healthTask = client.Cluster.HealthAsync(r => r.Level(Elastic.Clients.Elasticsearch.Level.Indices)); var statsTask = client.Cluster.StatsAsync(); - var catIndicesTask = client.Cat.IndicesAsync(r => r.Bytes(Elasticsearch.Net.Bytes.B)); - var catShardsTask = client.Cat.ShardsAsync(); - await Task.WhenAll(healthTask, statsTask, catIndicesTask, catShardsTask); + var indicesStatsTask = client.Indices.StatsAsync(); + await Task.WhenAll(healthTask, statsTask, indicesStatsTask); var healthResponse = await healthTask; var statsResponse = await statsTask; - var catIndicesResponse = await catIndicesTask; - var catShardsResponse = await catShardsTask; + var indicesStatsResponse = await indicesStatsTask; - if (!healthResponse.IsValid || !statsResponse.IsValid || !catIndicesResponse.IsValid || !catShardsResponse.IsValid) + if (!healthResponse.IsValidResponse || !statsResponse.IsValidResponse || !indicesStatsResponse.IsValidResponse) return Problem(title: "Elasticsearch cluster information is unavailable."); - // Count unassigned shards per index - var unassignedByIndex = (catShardsResponse.Records ?? []) - .Where(s => string.Equals(s.State, "UNASSIGNED", StringComparison.OrdinalIgnoreCase)) - .GroupBy(s => s.Index ?? String.Empty, StringComparer.OrdinalIgnoreCase) - .ToDictionary(g => g.Key, g => g.Count(), StringComparer.OrdinalIgnoreCase); - - var indexDetails = (catIndicesResponse.Records ?? []) - .OrderByDescending(i => long.TryParse(i.StoreSize, out var s) ? s : 0) - .Select(i => new ElasticsearchIndexDetailResponse( - Index: i.Index, - Health: i.Health, - Status: i.Status, - Primary: int.TryParse(i.Primary, out var p) ? p : 0, - Replica: int.TryParse(i.Replica, out var r) ? r : 0, - DocsCount: long.TryParse(i.DocsCount, out var dc) ? dc : 0, - StoreSizeInBytes: long.TryParse(i.StoreSize, out var ss) ? ss : 0, - UnassignedShards: unassignedByIndex.GetValueOrDefault(i.Index ?? String.Empty, 0) + // Count unassigned shards per index from health response + var unassignedByIndex = (healthResponse.Indices ?? new Dictionary()) + .Where(kvp => kvp.Value.UnassignedShards > 0) + .ToDictionary(kvp => kvp.Key, kvp => kvp.Value.UnassignedShards, StringComparer.OrdinalIgnoreCase); + + var indexDetails = (indicesStatsResponse.Indices ?? new Dictionary()) + .OrderByDescending(kvp => kvp.Value.Total?.Store?.SizeInBytes ?? 0) + .Select(kvp => new ElasticsearchIndexDetailResponse( + Index: kvp.Key, + Health: kvp.Value.Health?.ToString().ToLowerInvariant(), + Status: kvp.Value.Status?.ToString().ToLowerInvariant(), + Primary: healthResponse.Indices?.GetValueOrDefault(kvp.Key)?.NumberOfShards ?? 0, + Replica: healthResponse.Indices?.GetValueOrDefault(kvp.Key)?.NumberOfReplicas ?? 0, + DocsCount: kvp.Value.Total?.Docs?.Count ?? 0, + StoreSizeInBytes: kvp.Value.Total?.Store?.SizeInBytes ?? 0, + UnassignedShards: unassignedByIndex.GetValueOrDefault(kvp.Key, 0) )) .ToArray(); @@ -345,7 +342,7 @@ public async Task> GetElasticsearchInfoA ), Indices: new ElasticsearchIndicesResponse( Count: statsResponse.Indices.Count, - DocsCount: statsResponse.Indices.Documents.Count, + DocsCount: statsResponse.Indices.Docs.Count, StoreSizeInBytes: statsResponse.Indices.Store.SizeInBytes ), IndexDetails: indexDetails @@ -358,43 +355,40 @@ public async Task> GetElasticsearch var client = _configuration.Client; try { - var repositoryResponse = await client.Cat.RepositoriesAsync(); - if (!repositoryResponse.IsValid) + var repositoryResponse = await client.Snapshot.GetRepositoryAsync(); + if (!repositoryResponse.IsValidResponse) return Problem(title: "Snapshot repository information is unavailable."); - if (!(repositoryResponse.Records?.Any() ?? false)) + if (repositoryResponse.Repositories is null || !repositoryResponse.Repositories.Any()) return Ok(new ElasticsearchSnapshotsResponse([], [])); - var repositoryNames = repositoryResponse.Records - .Where(r => !String.IsNullOrEmpty(r.Id)) - .Select(r => r.Id!) - .ToArray(); + var repositoryNames = repositoryResponse.Repositories.Select(r => r.Key).ToArray(); var snapshotTasks = repositoryNames .Select(async repositoryName => { - var snapshotResponse = await client.Cat.SnapshotsAsync(r => r.RepositoryName(repositoryName)); - if (!snapshotResponse.IsValid) + var snapshotResponse = await client.Snapshot.GetAsync(repositoryName, "*"); + if (!snapshotResponse.IsValidResponse) return ( RepositoryName: repositoryName, Snapshots: Array.Empty(), Error: $"Unable to retrieve snapshots for repository: {repositoryName}." ); - var snapshotRecords = snapshotResponse.Records?.ToArray() ?? []; + var snapshots = snapshotResponse.Snapshots?.ToArray() ?? []; return ( RepositoryName: repositoryName, - Snapshots: snapshotRecords.Select(s => new ElasticsearchSnapshotResponse( + Snapshots: snapshots.Select(s => new ElasticsearchSnapshotResponse( Repository: repositoryName, - Name: s.Id ?? String.Empty, - Status: s.Status ?? String.Empty, - StartTime: s.StartEpoch > 0 ? DateTimeOffset.FromUnixTimeSeconds(s.StartEpoch).UtcDateTime : null, - EndTime: s.EndEpoch > 0 ? DateTimeOffset.FromUnixTimeSeconds(s.EndEpoch).UtcDateTime : null, + Name: s.Snapshot, + Status: s.State ?? String.Empty, + StartTime: s.StartTime?.UtcDateTime, + EndTime: s.EndTime?.UtcDateTime, Duration: s.Duration?.ToString() ?? String.Empty, - IndicesCount: s.Indices, - SuccessfulShards: s.SuccessfulShards, - FailedShards: s.FailedShards, - TotalShards: s.TotalShards + IndicesCount: s.Indices?.Count ?? 0, + SuccessfulShards: s.Shards?.Successful ?? 0, + FailedShards: s.Shards?.Failed ?? 0, + TotalShards: s.Shards?.Total ?? 0 )).ToArray(), Error: (string?)null ); diff --git a/src/Exceptionless.Web/Controllers/EventController.cs b/src/Exceptionless.Web/Controllers/EventController.cs index 90100c092d..e5ebaab077 100644 --- a/src/Exceptionless.Web/Controllers/EventController.cs +++ b/src/Exceptionless.Web/Controllers/EventController.cs @@ -26,10 +26,10 @@ using Foundatio.Repositories.Elasticsearch.Extensions; using Foundatio.Repositories.Extensions; using Foundatio.Repositories.Models; +using Foundatio.Serializer; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.Net.Http.Headers; -using Newtonsoft.Json; namespace Exceptionless.Web.Controllers; @@ -47,7 +47,7 @@ public class EventController : RepositoryApiController>> GetInternalAsync( Date = e.Date, Data = summaryData.Data }; - }).ToList(), events.HasMore && !NextPageExceedsSkipLimit(page, limit), page, events.Total, events.Hits.FirstOrDefault()?.GetSortToken(), events.Hits.LastOrDefault()?.GetSortToken()); + }).ToList(), events.HasMore && !NextPageExceedsSkipLimit(page, limit), page, events.Total, events.Hits.FirstOrDefault()?.GetSortToken(_serializer), events.Hits.LastOrDefault()?.GetSortToken(_serializer)); case "stack_recent": case "stack_frequent": case "stack_new": @@ -358,7 +358,7 @@ private async Task>> GetInternalAsync( return OkWithResourceLinks(summaries.Take(limit).ToList(), summaries.Count > limit && !NextPageExceedsSkipLimit(resolvedPage, limit), resolvedPage, total); default: events = await GetEventsInternalAsync(sf, ti, filter, sort, page, limit, before, after); - return OkWithResourceLinks(events.Documents.ToArray(), events.HasMore && !NextPageExceedsSkipLimit(page, limit), page, events.Total, events.Hits.FirstOrDefault()?.GetSortToken(), events.Hits.LastOrDefault()?.GetSortToken()); + return OkWithResourceLinks(events.Documents.ToArray(), events.HasMore && !NextPageExceedsSkipLimit(page, limit), page, events.Total, events.Hits.FirstOrDefault()?.GetSortToken(_serializer), events.Hits.LastOrDefault()?.GetSortToken(_serializer)); } } catch (ApplicationException ex) @@ -420,7 +420,7 @@ private Task> GetEventsInternalAsync(AppFilter sf, .Index(ti.Range.UtcStart, ti.Range.UtcEnd), o => page.HasValue ? o.PageNumber(page).PageLimit(limit) - : o.SearchBeforeToken(before).SearchAfterToken(after).PageLimit(limit)); + : o.SearchBeforeToken(before, _serializer).SearchAfterToken(after, _serializer).PageLimit(limit)); } /// @@ -1126,7 +1126,7 @@ private async Task GetSubmitEventAsync(string? projectId = null, i charSet = contentTypeHeader.Charset.ToString(); } - var stream = new MemoryStream(ev.GetBytes(_jsonSerializerSettings)); + using var stream = new MemoryStream(ev.GetBytes(_serializer)); await _eventPostService.EnqueueAsync(new EventPost(_appOptions.EnableArchive) { ApiVersion = apiVersion, diff --git a/src/Exceptionless.Web/Controllers/ProjectController.cs b/src/Exceptionless.Web/Controllers/ProjectController.cs index 90578a442d..557b1d33fb 100644 --- a/src/Exceptionless.Web/Controllers/ProjectController.cs +++ b/src/Exceptionless.Web/Controllers/ProjectController.cs @@ -17,6 +17,7 @@ using Foundatio.Queues; using Foundatio.Repositories; using Foundatio.Repositories.Models; +using Foundatio.Serializer; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using DataDictionary = Exceptionless.Core.Models.DataDictionary; @@ -35,6 +36,7 @@ public class ProjectController : RepositoryApiController _workItemQueue; private readonly BillingManager _billingManager; private readonly SlackService _slackService; + private readonly ITextSerializer _serializer; private readonly AppOptions _options; private readonly UsageService _usageService; private readonly SampleDataService _sampleDataService; @@ -51,6 +53,7 @@ public ProjectController( SampleDataService sampleDataService, ApiMapper mapper, IAppQueryValidator validator, + ITextSerializer serializer, AppOptions options, UsageService usageService, TimeProvider timeProvider, @@ -65,6 +68,7 @@ ILoggerFactory loggerFactory _workItemQueue = workItemQueue; _billingManager = billingManager; _slackService = slackService; + _serializer = serializer; _sampleDataService = sampleDataService; _options = options; _usageService = usageService; @@ -702,7 +706,7 @@ public async Task RemoveSlackAsync(string id) if (project is null) return NotFound(); - var token = project.GetSlackToken(); + var token = project.GetSlackToken(_serializer, _logger); using var _ = _logger.BeginScope(new ExceptionlessState().Property("Token", token).Tag("Slack").Identity(CurrentUser.EmailAddress).Property("User", CurrentUser).SetHttpContext(HttpContext)); if (token is not null) diff --git a/src/Exceptionless.Web/Controllers/SavedViewController.cs b/src/Exceptionless.Web/Controllers/SavedViewController.cs index 227bded5dc..9cd70647e7 100644 --- a/src/Exceptionless.Web/Controllers/SavedViewController.cs +++ b/src/Exceptionless.Web/Controllers/SavedViewController.cs @@ -17,7 +17,7 @@ namespace Exceptionless.App.Controllers.API; [Route(API_PREFIX + "/saved-views")] [Authorize(Policy = AuthorizationRoles.UserPolicy)] -public class SavedViewController : RepositoryApiController +public partial class SavedViewController : RepositoryApiController { private const int MaxViewsPerOrganization = 100; @@ -355,12 +355,12 @@ private async Task NameExistsAsync(string organizationId, string viewType, private static bool IsValidSlug(string slug) { - return Regex.IsMatch(slug, "^[a-z0-9]+(?:-[a-z0-9]+)*$") && !IsReservedSlug(slug); + return !IsReservedSlug(slug) && SavedView.SlugRegex().IsMatch(slug); } private static bool IsReservedSlug(string slug) { - return Regex.IsMatch(slug, "^[a-f0-9]{24}$"); + return ObjectIdSlugRegex().IsMatch(slug); } private static string ToSlug(string value) @@ -378,4 +378,7 @@ private static string ToFallbackSlug(string value, string id) return String.IsNullOrWhiteSpace(id) ? "saved-view" : $"saved-view-{id}"; } + + [GeneratedRegex("^[a-f0-9]{24}$")] + private static partial Regex ObjectIdSlugRegex(); } diff --git a/src/Exceptionless.Web/Controllers/StackController.cs b/src/Exceptionless.Web/Controllers/StackController.cs index 9b9b5630c4..eea8603e3e 100644 --- a/src/Exceptionless.Web/Controllers/StackController.cs +++ b/src/Exceptionless.Web/Controllers/StackController.cs @@ -599,7 +599,7 @@ private async Task> GetStackSummariesAsync(IColle return new List(); var systemFilter = new RepositoryQuery().AppFilter(eventSystemFilter).DateRange(ti.Range.UtcStart, ti.Range.UtcEnd, (PersistentEvent e) => e.Date).Index(ti.Range.UtcStart, ti.Range.UtcEnd); - var stackTerms = await _eventRepository.CountAsync(q => q.SystemFilter(systemFilter).FilterExpression(String.Join(" OR ", stacks.Select(r => $"stack:{r.Id}"))).AggregationsExpression($"terms:(stack_id~{stacks.Count} cardinality:user sum:count~1 min:date max:date)")); + var stackTerms = await _eventRepository.CountAsync(q => q.SystemFilter(systemFilter).Stack(stacks.Select(r => r.Id)).AggregationsExpression($"terms:(stack_id~{stacks.Count} cardinality:user sum:count~1 min:date max:date)")); var buckets = stackTerms.Aggregations.Terms("terms_stack_id")?.Buckets ?? []; return await GetStackSummariesAsync(stacks, buckets, eventSystemFilter, ti); } diff --git a/src/Exceptionless.Web/Exceptionless.Web.csproj b/src/Exceptionless.Web/Exceptionless.Web.csproj index 7f1fb15deb..388dd58d89 100644 --- a/src/Exceptionless.Web/Exceptionless.Web.csproj +++ b/src/Exceptionless.Web/Exceptionless.Web.csproj @@ -19,7 +19,6 @@ - @@ -84,4 +83,4 @@ - \ No newline at end of file + diff --git a/src/Exceptionless.Web/Hubs/WebSocketConnectionManager.cs b/src/Exceptionless.Web/Hubs/WebSocketConnectionManager.cs index 662aabff39..dfdd8747de 100644 --- a/src/Exceptionless.Web/Hubs/WebSocketConnectionManager.cs +++ b/src/Exceptionless.Web/Hubs/WebSocketConnectionManager.cs @@ -129,7 +129,8 @@ private Task SendMessageAsync(WebSocket socket, object message) try { - await socket.SendAsync(buffer: new ArraySegment(Encoding.ASCII.GetBytes(serializedMessage), 0, serializedMessage.Length), + byte[] bytes = Encoding.UTF8.GetBytes(serializedMessage); + await socket.SendAsync(buffer: new ArraySegment(bytes, 0, bytes.Length), messageType: WebSocketMessageType.Text, endOfMessage: true, cancellationToken: CancellationToken.None); diff --git a/src/Exceptionless.Web/Models/Auth/ExternalAuthInfo.cs b/src/Exceptionless.Web/Models/Auth/ExternalAuthInfo.cs index 128e8400a4..02d6b21c4e 100644 --- a/src/Exceptionless.Web/Models/Auth/ExternalAuthInfo.cs +++ b/src/Exceptionless.Web/Models/Auth/ExternalAuthInfo.cs @@ -3,7 +3,8 @@ namespace Exceptionless.Web.Models; -// NOTE: This will bypass our LowerCaseUnderscorePropertyNamesContractResolver and provide the correct casing. +// NOTE: Explicit [JsonPropertyName] attributes ensure camelCase keys for these properties, +// overriding the global SnakeCaseLower naming policy. public record ExternalAuthInfo { [Required] diff --git a/src/Exceptionless.Web/Models/SavedView/NewSavedView.cs b/src/Exceptionless.Web/Models/SavedView/NewSavedView.cs index 999335e0f3..68b648218b 100644 --- a/src/Exceptionless.Web/Models/SavedView/NewSavedView.cs +++ b/src/Exceptionless.Web/Models/SavedView/NewSavedView.cs @@ -43,7 +43,7 @@ public record NewSavedView : IOwnedByOrganization, IValidatableObject public string? Sort { get; set; } [MaxLength(100)] - [RegularExpression("^[a-z0-9]+(?:-[a-z0-9]+)*$")] + [RegularExpression(SavedView.SlugPattern)] public string? Slug { get; set; } [Required] diff --git a/src/Exceptionless.Web/Models/SavedView/UpdateSavedView.cs b/src/Exceptionless.Web/Models/SavedView/UpdateSavedView.cs index 07503cc20f..567981d765 100644 --- a/src/Exceptionless.Web/Models/SavedView/UpdateSavedView.cs +++ b/src/Exceptionless.Web/Models/SavedView/UpdateSavedView.cs @@ -14,7 +14,7 @@ public class UpdateSavedView : IValidatableObject [MaxLength(100)] public string? Sort { get; set; } [MaxLength(100)] - [RegularExpression("^[a-z0-9]+(?:-[a-z0-9]+)*$")] + [RegularExpression(SavedView.SlugPattern)] public string? Slug { get; set; } [MaxLength(SavedView.MaxFilterDefinitionsLength)] public string? FilterDefinitions { get; set; } diff --git a/src/Exceptionless.Web/Startup.cs b/src/Exceptionless.Web/Startup.cs index 4a37df1963..269646ba89 100644 --- a/src/Exceptionless.Web/Startup.cs +++ b/src/Exceptionless.Web/Startup.cs @@ -1,5 +1,6 @@ using System.Diagnostics; using System.Security.Claims; +using System.Text.Json; using Exceptionless.Core; using Exceptionless.Core.Authorization; using Exceptionless.Core.Extensions; @@ -58,19 +59,19 @@ public void ConfigureServices(IServiceCollection services) services.AddControllers(o => { o.ModelBinderProviders.Insert(0, new CustomAttributesModelBinderProvider()); - o.ModelMetadataDetailsProviders.Add(new SystemTextJsonValidationMetadataProvider(LowerCaseUnderscoreNamingPolicy.Instance)); + o.ModelMetadataDetailsProviders.Add(new SystemTextJsonValidationMetadataProvider(JsonNamingPolicy.SnakeCaseLower)); o.InputFormatters.Insert(0, new RawRequestBodyFormatter()); }) .AddJsonOptions(o => { - o.JsonSerializerOptions.ConfigureExceptionlessDefaults(); + o.JsonSerializerOptions.ConfigureExceptionlessApiDefaults(); o.JsonSerializerOptions.Converters.Add(new DeltaJsonConverterFactory()); }); // Have to add this to get the open api json file to be snake case. services.ConfigureHttpJsonOptions(o => { - o.SerializerOptions.ConfigureExceptionlessDefaults(); + o.SerializerOptions.ConfigureExceptionlessApiDefaults(); o.SerializerOptions.Converters.Add(new DeltaJsonConverterFactory()); }); @@ -138,7 +139,8 @@ public void ConfigureServices(IServiceCollection services) private void CustomizeProblemDetails(ProblemDetailsContext ctx) { - ctx.ProblemDetails.Extensions.Add("instance", $"{ctx.HttpContext.Request.Method} {ctx.HttpContext.Request.Path}"); + ctx.ProblemDetails.Instance = $"{ctx.HttpContext.Request.Method} {ctx.HttpContext.Request.Path}"; + if (ctx.HttpContext.Items.TryGetValue("reference-id", out object? refId) && refId is string referenceId) { ctx.ProblemDetails.Extensions.Add("reference-id", referenceId); @@ -151,7 +153,7 @@ private void CustomizeProblemDetails(ProblemDetailsContext ctx) if (ctx.ProblemDetails is ValidationProblemDetails validationProblem) { - // This might be possible to accomplish via serializer. + // MVC validation keys are CLR property names; normalize them to the API's snake_case contract. // NOTE: the key could be wrong for things like ExternalAuthInfo where the keys are camel case. validationProblem.Errors = validationProblem.Errors .ToDictionary( @@ -160,8 +162,6 @@ private void CustomizeProblemDetails(ProblemDetailsContext ctx) ); } - // errors - // TODO: Check casing of property names of model state validation errors. } public void Configure(IApplicationBuilder app) diff --git a/src/Exceptionless.Web/Utility/Delta/Delta.cs b/src/Exceptionless.Web/Utility/Delta/Delta.cs index 38c402227e..807cc05248 100644 --- a/src/Exceptionless.Web/Utility/Delta/Delta.cs +++ b/src/Exceptionless.Web/Utility/Delta/Delta.cs @@ -22,6 +22,7 @@ public class Delta : DynamicObject /*, IDelta */ where TEntityType private HashSet _changedProperties = null!; private TEntityType _entity = null!; private Type _entityType = null!; + private JsonSerializerOptions? _options; /// /// Initializes a new instance of . @@ -33,12 +34,9 @@ public Delta() : this(typeof(TEntityType)) /// /// Initializes a new instance of . /// - /// - /// The derived entity type for which the changes would be tracked. - /// should be assignable to instances of . - /// - public Delta(Type entityType) + public Delta(Type entityType, JsonSerializerOptions? options = null) { + _options = options; Initialize(entityType); } @@ -70,11 +68,9 @@ public bool TrySetPropertyValue(string name, object? value, TEntityType? target { ArgumentNullException.ThrowIfNull(name); - if (!_propertiesThatExist.ContainsKey(name)) + if (!_propertiesThatExist.TryGetValue(name, out var cacheHit)) return false; - var cacheHit = _propertiesThatExist[name]; - if (value is null && !IsNullable(cacheHit.MemberType)) return false; @@ -84,7 +80,9 @@ public bool TrySetPropertyValue(string name, object? value, TEntityType? target { try { - value = JsonSerializer.Deserialize(jsonElement.GetRawText(), cacheHit.MemberType); + value = _options is not null + ? JsonSerializer.Deserialize(jsonElement.GetRawText(), cacheHit.MemberType, _options) + : JsonSerializer.Deserialize(jsonElement.GetRawText(), cacheHit.MemberType); } catch (Exception) { diff --git a/src/Exceptionless.Web/Utility/Delta/DeltaJsonConverter.cs b/src/Exceptionless.Web/Utility/Delta/DeltaJsonConverter.cs index 68e5f729a0..2e71eb7fc5 100644 --- a/src/Exceptionless.Web/Utility/Delta/DeltaJsonConverter.cs +++ b/src/Exceptionless.Web/Utility/Delta/DeltaJsonConverter.cs @@ -1,3 +1,4 @@ +using System.Reflection; using System.Text.Json; using System.Text.Json.Serialization; @@ -33,20 +34,31 @@ public override bool CanConvert(Type typeToConvert) public class DeltaJsonConverter : JsonConverter> where TEntityType : class { private readonly JsonSerializerOptions _options; - private readonly Dictionary _jsonNameToPropertyName; + private readonly Dictionary _propertyNameMap; public DeltaJsonConverter(JsonSerializerOptions options) { - // Create a copy without the converter to avoid infinite recursion + // Create a copy without the factory to avoid infinite recursion if a Delta property + // ever appears inside another Delta model. _options = new JsonSerializerOptions(options); + for (int i = _options.Converters.Count - 1; i >= 0; i--) + { + if (_options.Converters[i] is DeltaJsonConverterFactory) + { + _options.Converters.RemoveAt(i); + break; + } + } // Build a mapping from JSON property names (snake_case) to C# property names (PascalCase) - _jsonNameToPropertyName = new Dictionary(StringComparer.OrdinalIgnoreCase); + _propertyNameMap = new Dictionary(StringComparer.OrdinalIgnoreCase); var entityType = typeof(TEntityType); foreach (var prop in entityType.GetProperties()) { - var jsonName = options.PropertyNamingPolicy?.ConvertName(prop.Name) ?? prop.Name; - _jsonNameToPropertyName[jsonName] = prop.Name; + // [JsonPropertyName] takes precedence over the naming policy + var jsonPropertyNameAttr = prop.GetCustomAttribute(); + var jsonName = jsonPropertyNameAttr?.Name ?? options.PropertyNamingPolicy?.ConvertName(prop.Name) ?? prop.Name; + _propertyNameMap[jsonName] = prop.Name; } } @@ -62,7 +74,7 @@ public DeltaJsonConverter(JsonSerializerOptions options) throw new JsonException("Expected StartObject token"); } - var delta = new Delta(); + var delta = new Delta(typeof(TEntityType), _options); while (reader.Read()) { @@ -85,7 +97,7 @@ public DeltaJsonConverter(JsonSerializerOptions options) reader.Read(); // Convert JSON property name (snake_case) to C# property name (PascalCase) - var propertyName = _jsonNameToPropertyName.TryGetValue(jsonPropertyName, out var mapped) + var propertyName = _propertyNameMap.TryGetValue(jsonPropertyName, out var mapped) ? mapped : jsonPropertyName; diff --git a/tests/Exceptionless.Tests/Controllers/AdminControllerTests.cs b/tests/Exceptionless.Tests/Controllers/AdminControllerTests.cs index ff8ab13fee..9777c4c00f 100644 --- a/tests/Exceptionless.Tests/Controllers/AdminControllerTests.cs +++ b/tests/Exceptionless.Tests/Controllers/AdminControllerTests.cs @@ -9,7 +9,6 @@ using Foundatio.Jobs; using Foundatio.Queues; using Foundatio.Repositories; -using Foundatio.Repositories.Migrations; using Foundatio.Repositories.Models; using Foundatio.Repositories.Utility; using Xunit; @@ -506,6 +505,7 @@ public async Task GetElasticsearch_AsGlobalAdmin_IndexDetailsContainExpectedFiel // Assert Assert.NotNull(elasticsearch); + Assert.NotNull(elasticsearch.IndexDetails); Assert.All(elasticsearch.IndexDetails, indexDetail => { Assert.True(indexDetail.DocsCount >= 0); @@ -556,7 +556,7 @@ public async Task GetSettings_AsGlobalAdmin_ReturnsAppOptions() // Assert Assert.NotNull(options); - Assert.True(options.ContainsKey("base_u_r_l")); + Assert.True(options.ContainsKey("base_url")); Assert.True(options.ContainsKey("app_mode")); } diff --git a/tests/Exceptionless.Tests/Controllers/Data/event-serialization-response.json b/tests/Exceptionless.Tests/Controllers/Data/event-serialization-response.json index f6ee5af089..adc282bab6 100644 --- a/tests/Exceptionless.Tests/Controllers/Data/event-serialization-response.json +++ b/tests/Exceptionless.Tests/Controllers/Data/event-serialization-response.json @@ -4,7 +4,7 @@ "project_id": "537650f3b77efe23a47914f4", "stack_id": "", "is_first_occurrence": true, - "created_utc": "2026-01-15T12:00:00", + "created_utc": "2026-01-15T12:00:00Z", "type": "error", "date": "2026-01-15T12:00:00+00:00", "tags": ["test", "serialization"], diff --git a/tests/Exceptionless.Tests/Controllers/Data/openapi.json b/tests/Exceptionless.Tests/Controllers/Data/openapi.json index a036591468..7a401caaf0 100644 --- a/tests/Exceptionless.Tests/Controllers/Data/openapi.json +++ b/tests/Exceptionless.Tests/Controllers/Data/openapi.json @@ -7901,7 +7901,7 @@ }, "slug": { "maxLength": 100, - "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$", + "pattern": "^(?![a-f0-9]{24}$)[a-z0-9]+(?:-[a-z0-9]+)*$", "type": [ "null", "string" diff --git a/tests/Exceptionless.Tests/Controllers/EventControllerTests.Casing.cs b/tests/Exceptionless.Tests/Controllers/EventControllerTests.Casing.cs new file mode 100644 index 0000000000..9d3a4aa601 --- /dev/null +++ b/tests/Exceptionless.Tests/Controllers/EventControllerTests.Casing.cs @@ -0,0 +1,316 @@ +using System.Text.Json; +using Exceptionless.Core.Jobs; +using Exceptionless.Core.Models; +using Exceptionless.Tests.Extensions; +using Exceptionless.Tests.Utility; +using Foundatio.Jobs; +using Foundatio.Repositories; +using Xunit; + +namespace Exceptionless.Tests.Controllers; + +/// +/// Integration tests that verify events submitted with different JSON casing conventions +/// are processed correctly through the full pipeline (API → Queue → Job → ES). +/// Reproduces critical issues found in the serialization audit diff. +/// +public partial class EventControllerTests +{ + private static readonly DateTimeOffset TestUtcNow = new(2026, 1, 15, 12, 0, 0, TimeSpan.Zero); + + private DateTimeOffset GetRecentDate() + { + TimeProvider.SetUtcNow(TestUtcNow); + return TestUtcNow.AddHours(-1); + } + + private static string FormatDate(DateTimeOffset date) => date.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss+00:00"); + + [Fact] + public async Task PostEvent_PascalCase_PreservesAllProperties() + { + // Arrange + var recentDate = GetRecentDate(); + string recentDateStr = FormatDate(recentDate); + string requestJson = $$""" + { + "Type": "error", + "Message": "PascalCase error test", + "Tags": ["integration", "pascal"], + "ReferenceId": "pascal-int-001", + "Count": 3, + "Value": 99.9, + "Geo": "51.5074,-0.1278", + "Date": "{{recentDateStr}}", + "@simple_error": { + "Message": "Something broke", + "Type": "System.Exception", + "StackTrace": " at Test.Run()" + } + } + """; + + // Act + await SendRequestAsync(r => r + .Post() + .AsTestOrganizationClientUser() + .AppendPath("events") + .Content(requestJson, "application/json") + .StatusCodeShouldBeAccepted() + ); + + var job = GetService(); + await job.RunAsync(TestCancellationToken); + await RefreshDataAsync(); + + // Assert + var events = await _eventRepository.GetAllAsync(); + var ev = events.Documents.FirstOrDefault(e => + String.Equals(e.ReferenceId, "pascal-int-001", StringComparison.OrdinalIgnoreCase)); + + Assert.NotNull(ev); + Assert.Equal("error", ev.Type); + Assert.Equal("PascalCase error test", ev.Message); + Assert.Equal(3, ev.Count); + Assert.Equal(99.9m, ev.Value); + Assert.Equal(recentDate.Date, ev.Date.Date); + Assert.Contains("integration", ev.Tags!); + Assert.Contains("pascal", ev.Tags!); + } + + [Fact] + public async Task PostEvent_CamelCase_PreservesAllProperties() + { + // Arrange + var recentDate = GetRecentDate(); + string recentDateStr = FormatDate(recentDate); + string requestJson = $$""" + { + "type": "error", + "message": "camelCase error test", + "tags": ["integration", "camel"], + "referenceId": "camel-int-001", + "count": 2, + "value": 55.5, + "geo": "48.8566,2.3522", + "date": "{{recentDateStr}}", + "@simple_error": { + "message": "Something broke", + "type": "System.Exception", + "stackTrace": " at Test.Run()" + } + } + """; + + // Act + await SendRequestAsync(r => r + .Post() + .AsTestOrganizationClientUser() + .AppendPath("events") + .Content(requestJson, "application/json") + .StatusCodeShouldBeAccepted() + ); + + var job = GetService(); + await job.RunAsync(TestCancellationToken); + await RefreshDataAsync(); + + // Assert + var events = await _eventRepository.GetAllAsync(); + var ev = events.Documents.FirstOrDefault(e => + String.Equals(e.ReferenceId, "camel-int-001", StringComparison.OrdinalIgnoreCase)); + + Assert.NotNull(ev); + Assert.Equal("error", ev.Type); + Assert.Equal("camelCase error test", ev.Message); + Assert.Equal(2, ev.Count); + Assert.Equal(55.5m, ev.Value); + Assert.Equal(recentDate.Date, ev.Date.Date); + Assert.Contains("integration", ev.Tags!); + Assert.Contains("camel", ev.Tags!); + } + + [Fact] + public async Task PostEvent_MixedCase_PreservesAllProperties() + { + // Arrange + var recentDate = GetRecentDate(); + string recentDateStr = FormatDate(recentDate); + string requestJson = $$""" + { + "TYPE": "error", + "message": "Mixed CASE error test", + "TAGS": ["integration", "mixed"], + "reference_id": "mixed-int-001", + "Count": 1, + "value": 10.0, + "GEO": "35.6762,139.6503", + "Date": "{{recentDateStr}}", + "@simple_error": { + "message": "Mixed case exception", + "type": "System.InvalidOperationException" + } + } + """; + + // Act + await SendRequestAsync(r => r + .Post() + .AsTestOrganizationClientUser() + .AppendPath("events") + .Content(requestJson, "application/json") + .StatusCodeShouldBeAccepted() + ); + + var job = GetService(); + await job.RunAsync(TestCancellationToken); + await RefreshDataAsync(); + + // Assert + var events = await _eventRepository.GetAllAsync(); + var ev = events.Documents.FirstOrDefault(e => + String.Equals(e.ReferenceId, "mixed-int-001", StringComparison.OrdinalIgnoreCase)); + + Assert.NotNull(ev); + Assert.Equal("error", ev.Type); + Assert.Equal("Mixed CASE error test", ev.Message); + Assert.Equal(1, ev.Count); + Assert.Equal(10.0m, ev.Value); + Assert.Equal(recentDate.Date, ev.Date.Date); + Assert.Contains("integration", ev.Tags!); + Assert.Contains("mixed", ev.Tags!); + } + + [Fact] + public async Task PostEvent_DateOnlyInData_PreservedAsString() + { + // Arrange + var recentDate = GetRecentDate(); + string recentDateStr = FormatDate(recentDate); + string fullDateStr = recentDate.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ"); + string requestJson = $$""" + { + "type": "log", + "message": "Date-only test", + "reference_id": "date-only-001", + "date": "{{recentDateStr}}", + "data": { + "date_only": "2026-01-15", + "full_date": "{{fullDateStr}}" + } + } + """; + + // Act + await SendRequestAsync(r => r + .Post() + .AsTestOrganizationClientUser() + .AppendPath("events") + .Content(requestJson, "application/json") + .StatusCodeShouldBeAccepted() + ); + + var job = GetService(); + await job.RunAsync(TestCancellationToken); + await RefreshDataAsync(); + + // Assert + var events = await _eventRepository.GetAllAsync(); + var ev = events.Documents.FirstOrDefault(e => + String.Equals(e.ReferenceId, "date-only-001", StringComparison.OrdinalIgnoreCase)); + + Assert.NotNull(ev); + Assert.Equal(recentDate.Date, ev.Date.Date); + Assert.NotNull(ev.Data); + Assert.True(ev.Data.TryGetValue("date_only", out var dateOnly)); + + Assert.IsType(dateOnly); + Assert.Equal("2026-01-15", dateOnly); + } + + [Fact] + public async Task PostEvent_PascalCase_ApiResponseHasCorrectFormat() + { + // Arrange + var recentDate = GetRecentDate(); + string recentDateStr = FormatDate(recentDate); + string requestJson = $$""" + { + "Type": "log", + "Message": "Response format test", + "ReferenceId": "format-001", + "Date": "{{recentDateStr}}", + "Value": 0, + "@user": { + "Identity": "pascal-user@example.com", + "Name": "Pascal User", + "Data": { + "PlanName": "premium" + } + } + } + """; + + // Act + await SendRequestAsync(r => r + .Post() + .AsTestOrganizationClientUser() + .AppendPath("events") + .Content(requestJson, "application/json") + .StatusCodeShouldBeAccepted() + ); + + var job = GetService(); + await job.RunAsync(TestCancellationToken); + await RefreshDataAsync(); + + var events = await _eventRepository.GetAllAsync(); + var ev = events.Documents.FirstOrDefault(e => + String.Equals(e.ReferenceId, "format-001", StringComparison.OrdinalIgnoreCase)); + Assert.NotNull(ev); + + var response = await SendRequestAsync(r => r + .AsGlobalAdminUser() + .AppendPath("events") + .AppendPath(ev.Id) + .StatusCodeShouldBeOk() + ); + + string responseJson = await response.Content.ReadAsStringAsync(TestCancellationToken); + using var doc = JsonDocument.Parse(responseJson); + + // Assert + Assert.True(doc.RootElement.TryGetProperty("type", out var typeProp)); + Assert.Equal("log", typeProp.GetString()); + + Assert.True(doc.RootElement.TryGetProperty("message", out var msgProp)); + Assert.Equal("Response format test", msgProp.GetString()); + + Assert.True(doc.RootElement.TryGetProperty("reference_id", out var refProp)); + Assert.Equal("format-001", refProp.GetString()); + + Assert.True(doc.RootElement.TryGetProperty("date", out var dateProp)); + Assert.NotNull(dateProp.GetString()); + Assert.True(DateTimeOffset.TryParse(dateProp.GetString(), out var parsedDate)); + Assert.Equal(recentDate.Date, parsedDate.Date); + + Assert.True(doc.RootElement.TryGetProperty("created_utc", out var createdUtcProp)); + Assert.True(createdUtcProp.GetString()?.EndsWith("Z", StringComparison.Ordinal) ?? false); + + Assert.True(doc.RootElement.TryGetProperty("data", out var dataProp)); + Assert.True(dataProp.TryGetProperty("@user", out var userProp)); + Assert.True(userProp.TryGetProperty("Identity", out var identityProp)); + Assert.Equal("pascal-user@example.com", identityProp.GetString()); + Assert.True(userProp.TryGetProperty("Name", out var nameProp)); + Assert.Equal("Pascal User", nameProp.GetString()); + Assert.True(userProp.TryGetProperty("Data", out var userDataProp)); + Assert.True(userDataProp.TryGetProperty("PlanName", out var planNameProp)); + Assert.Equal("premium", planNameProp.GetString()); + Assert.False(userProp.TryGetProperty("identity", out _)); + Assert.False(userProp.TryGetProperty("name", out _)); + Assert.False(userProp.TryGetProperty("data", out _)); + + Assert.True(doc.RootElement.TryGetProperty("value", out var valueProp)); + Assert.Equal("0", valueProp.GetRawText()); + } +} diff --git a/tests/Exceptionless.Tests/Controllers/EventControllerTests.cs b/tests/Exceptionless.Tests/Controllers/EventControllerTests.cs index 0f09c53ee2..d3d563809d 100644 --- a/tests/Exceptionless.Tests/Controllers/EventControllerTests.cs +++ b/tests/Exceptionless.Tests/Controllers/EventControllerTests.cs @@ -25,6 +25,7 @@ using Foundatio.Queues; using Foundatio.Repositories; using Foundatio.Repositories.Models; +using Foundatio.Serializer; using Microsoft.Net.Http.Headers; using Xunit; using MediaTypeHeaderValue = System.Net.Http.Headers.MediaTypeHeaderValue; @@ -32,7 +33,7 @@ namespace Exceptionless.Tests.Controllers; -public class EventControllerTests : IntegrationTestsBase +public partial class EventControllerTests : IntegrationTestsBase { private readonly JsonSerializerOptions _jsonSerializerOptions; private readonly IOrganizationRepository _organizationRepository; @@ -70,7 +71,7 @@ protected override async Task ResetDataAsync() [Fact] public async Task PostEvent_WithValidPayload_EnqueuesAndProcessesEventAsync() { - var jsonOptions = GetService(); + var serializer = GetService(); /* language=json */ const string json = """{"message":"test","reference_id":"TestReferenceId","@user":{"identity":"Test user","name":null}}"""; await SendRequestAsync(r => r @@ -97,12 +98,11 @@ await SendRequestAsync(r => r Assert.Equal("test", ev.Message); Assert.Equal("TestReferenceId", ev.ReferenceId); - var identity = ev.GetUserIdentity(jsonOptions); + var identity = ev.GetUserIdentity(serializer, _logger); Assert.NotNull(identity); Assert.Equal("Test user", identity.Identity); Assert.Null(identity.Name); - Assert.Null(identity.Name); - Assert.Null(ev.GetUserDescription(jsonOptions)); + Assert.Null(ev.GetUserDescription(serializer, _logger)); // post description await _eventUserDescriptionQueue.DeleteQueueAsync(); @@ -128,13 +128,12 @@ await SendRequestAsync(r => r ev = await _eventRepository.GetByIdAsync(ev.Id); Assert.NotNull(ev); - identity = ev.GetUserIdentity(jsonOptions); + identity = ev.GetUserIdentity(serializer, _logger); Assert.NotNull(identity); Assert.Equal("Test user", identity.Identity); Assert.Null(identity.Name); - Assert.Null(identity.Name); - var description = ev.GetUserDescription(jsonOptions); + var description = ev.GetUserDescription(serializer, _logger); Assert.NotNull(description); Assert.Equal("Test Description", description.Description); Assert.Equal(TestConstants.UserEmail, description.EmailAddress); @@ -229,7 +228,7 @@ await SendRequestAsync(r => r Assert.NotNull(ev.Data); Assert.Equal("custom value", ev.Data["custom_property"]); - var identity = ev.GetUserIdentity(_jsonSerializerOptions); + var identity = ev.GetUserIdentity(GetService(), Log.CreateLogger()); Assert.NotNull(identity); Assert.Equal("user-123", identity.Identity); Assert.Equal("Test User", identity.Name); @@ -316,7 +315,7 @@ public async Task CanPostCompressedStringAsync() [Fact] public async Task CanPostJsonWithUserInfoAsync() { - var jsonOptions = GetService(); + var serializer = GetService(); /* language=json */ const string json = """{"message":"test","@user":{"identity":"Test user","name":null}}"""; await SendRequestAsync(r => r @@ -342,7 +341,7 @@ await SendRequestAsync(r => r var ev = events.Documents.Single(e => String.Equals(e.Type, Event.KnownTypes.Log)); Assert.Equal("test", ev.Message); - var userInfo = ev.GetUserIdentity(jsonOptions); + var userInfo = ev.GetUserIdentity(serializer, _logger); Assert.NotNull(userInfo); Assert.Equal("Test user", userInfo.Identity); Assert.Null(userInfo.Name); @@ -897,6 +896,7 @@ await CreateDataAsync(d => .TestProject() .Message("New stack - skip due to date filter") .Type(Event.KnownTypes.Log) + .Source("skip-due-to-date-filter") .Status(StackStatus.Open) .TotalOccurrences(50) .IsFirstOccurrence() @@ -907,6 +907,7 @@ await CreateDataAsync(d => .TestProject() .Message("Old stack - new event") .Type(Event.KnownTypes.Log) + .Source("old-stack-new-event") .Status(StackStatus.Regressed) .TotalOccurrences(33) .FirstOccurrence(utcNow.SubtractYears(1)) @@ -916,6 +917,7 @@ await CreateDataAsync(d => .TestProject() .Message("New Stack - event not marked as first occurrence") .Type(Event.KnownTypes.Log) + .Source("new-stack-not-first-occurrence") .Status(StackStatus.Open) .TotalOccurrences(15) .FirstOccurrence(utcNow.SubtractDays(2)) @@ -1740,7 +1742,7 @@ await SendRequestAsync(r => r .Replace("", processedEvent.Id) .Replace("", processedEvent.StackId); - Assert.Equal(ToPrettyJson(expectedJson), ToPrettyJson(actualJson)); + JsonAssert.AssertJsonEqualsNormalized(expectedJson, actualJson, _jsonSerializerOptions); } [Fact] @@ -1782,14 +1784,11 @@ await SendRequestAsync(r => r Assert.Equal("log", ev.Type); Assert.Equal("Test with extra properties", ev.Message); - // Note: Extra root properties should be captured if JsonExtensionData is implemented on Event class - // If not implemented, this assertion verifies the current behavior - if (ev.Data is not null && ev.Data.ContainsKey("custom_field")) - { - Assert.Equal("custom_value", ev.Data["custom_field"]); - Assert.Equal(42L, ev.Data["custom_number"]); - Assert.True(ev.Data["custom_flag"] as bool?); - } + // Extra root properties are captured via [JsonExtensionData] + OnDeserialized merge into Data + Assert.NotNull(ev.Data); + Assert.Equal("custom_value", ev.Data["custom_field"]); + Assert.Equal(42L, ev.Data["custom_number"]); + Assert.Equal(true, ev.Data["custom_flag"]); } [Fact] @@ -1824,7 +1823,7 @@ await SendRequestAsync(r => r await processEventsJob.RunAsync(TestCancellationToken); await RefreshDataAsync(); - var jsonOptions = GetService(); + var serializer = GetService(); // Assert var events = await _eventRepository.GetAllAsync(); @@ -1834,7 +1833,7 @@ await SendRequestAsync(r => r Assert.Equal("Error with mixed data", ev.Message); // Verify known data is properly deserialized - var userInfo = ev.GetUserIdentity(jsonOptions); + var userInfo = ev.GetUserIdentity(serializer, _logger); Assert.NotNull(userInfo); Assert.Equal("user@example.com", userInfo.Identity); Assert.Equal("Test User", userInfo.Name); @@ -1843,12 +1842,10 @@ await SendRequestAsync(r => r string? version = ev.GetVersion(); Assert.Equal("1.0.0", version); - // Verify extra properties are captured if JsonExtensionData is implemented - if (ev.Data is not null && ev.Data.TryGetValue("extra_field_1", out object? value)) - { - Assert.Equal("value1", value); - Assert.Equal(99L, ev.Data["extra_field_2"]); - } + // Extra root properties are captured via [JsonExtensionData] + OnDeserialized merge into Data + Assert.NotNull(ev.Data); + Assert.Equal("value1", ev.Data["extra_field_1"]); + Assert.Equal(99L, ev.Data["extra_field_2"]); } [Fact] @@ -1890,10 +1887,24 @@ await SendRequestAsync(r => r Assert.Equal("log", ev.Type); Assert.Equal("Test with complex properties", ev.Message); - - // Verify event was processed successfully - Assert.NotNull(ev.Id); Assert.NotEqual(DateTimeOffset.MinValue, ev.Date); + Assert.InRange(ev.Date, DateTimeOffset.UtcNow.AddMinutes(-1), DateTimeOffset.UtcNow.AddMinutes(1)); + + // Verify complex properties are captured in Data via JsonExtensionData + OnDeserialized + Assert.NotNull(ev.Data); + // Verify nested object structure is preserved + Assert.True(ev.Data.TryGetValue("metadata", out var metadataRaw), "metadata key should be captured in Data"); + var metadata = Assert.IsType>(metadataRaw); + Assert.Equal("value1", metadata["key1"]); + Assert.Equal(42L, metadata["key2"]); + var nested = Assert.IsType>(metadata["nested"]); + Assert.Equal("value", nested["inner"]); + + // Verify array structure is preserved + Assert.True(ev.Data.TryGetValue("tags_list", out var tagsListRaw), "tags_list key should be captured in Data"); + var tagsList = Assert.IsType>(tagsListRaw); + Assert.Equal(3, tagsList.Count); + Assert.Equal("tag1", tagsList[0]); } [Fact] @@ -1933,13 +1944,4 @@ await SendRequestAsync(r => r Assert.Equal("ref-1234567890", ev.ReferenceId); } - private string ToPrettyJson(string json) - { - using var document = JsonDocument.Parse(json); - var prettyJsonOptions = new JsonSerializerOptions(_jsonSerializerOptions) - { - WriteIndented = true - }; - return JsonSerializer.Serialize(document.RootElement, prettyJsonOptions); - } } diff --git a/tests/Exceptionless.Tests/Controllers/OpenApiControllerTests.cs b/tests/Exceptionless.Tests/Controllers/OpenApiControllerTests.cs index 23e1a98cfc..0fd4940d42 100644 --- a/tests/Exceptionless.Tests/Controllers/OpenApiControllerTests.cs +++ b/tests/Exceptionless.Tests/Controllers/OpenApiControllerTests.cs @@ -25,9 +25,17 @@ public async Task GetOpenApiJson_Default_ReturnsExpectedBaseline() string actualJson = await response.Content.ReadAsStringAsync(TestCancellationToken); // Assert - string expectedJson = (await File.ReadAllTextAsync(baselinePath, TestCancellationToken)).Replace("\\r\\n", "\\n"); - actualJson = actualJson.Replace("\\r\\n", "\\n"); + string expectedJson = NormalizeOpenApiJson(await File.ReadAllTextAsync(baselinePath, TestCancellationToken)); + actualJson = NormalizeOpenApiJson(actualJson); Assert.Equal(expectedJson, actualJson); } + + private static string NormalizeOpenApiJson(string json) + { + return json + .ReplaceLineEndings("\n") + .Replace("\\r\\n", "\\n") + .TrimEnd('\n'); + } } diff --git a/tests/Exceptionless.Tests/Controllers/SavedViewControllerTests.cs b/tests/Exceptionless.Tests/Controllers/SavedViewControllerTests.cs index a479dcd3f0..bacae5cd2d 100644 --- a/tests/Exceptionless.Tests/Controllers/SavedViewControllerTests.cs +++ b/tests/Exceptionless.Tests/Controllers/SavedViewControllerTests.cs @@ -187,6 +187,7 @@ public async Task PostAsync_AsOrganizationUser_CanCreateSavedView() [Fact] public Task PostAsync_WithEmptyName_ReturnsUnprocessableEntity() { + // Arrange var newView = new NewSavedView { OrganizationId = SampleDataService.TEST_ORG_ID, @@ -195,6 +196,7 @@ public Task PostAsync_WithEmptyName_ReturnsUnprocessableEntity() ViewType = "events" }; + // Act & Assert return SendRequestAsync(r => r .Post() .AsGlobalAdminUser() @@ -207,6 +209,7 @@ public Task PostAsync_WithEmptyName_ReturnsUnprocessableEntity() [Fact] public Task PostAsync_WithNameThatCannotCreateUrlName_ReturnsUnprocessableEntity() { + // Arrange var newView = new NewSavedView { OrganizationId = SampleDataService.TEST_ORG_ID, @@ -215,6 +218,7 @@ public Task PostAsync_WithNameThatCannotCreateUrlName_ReturnsUnprocessableEntity ViewType = "events" }; + // Act & Assert return SendRequestAsync(r => r .Post() .AsGlobalAdminUser() @@ -227,6 +231,7 @@ public Task PostAsync_WithNameThatCannotCreateUrlName_ReturnsUnprocessableEntity [Fact] public Task PostAsync_WithObjectIdUrlName_ReturnsUnprocessableEntity() { + // Arrange var newView = new NewSavedView { OrganizationId = SampleDataService.TEST_ORG_ID, @@ -236,6 +241,7 @@ public Task PostAsync_WithObjectIdUrlName_ReturnsUnprocessableEntity() ViewType = "events" }; + // Act & Assert return SendRequestAsync(r => r .Post() .AsGlobalAdminUser() @@ -248,9 +254,11 @@ public Task PostAsync_WithObjectIdUrlName_ReturnsUnprocessableEntity() [Fact] public async Task PostAsync_DuplicateName_ReturnsConflict() { + // Arrange var created = await CreateSavedViewAsync("Duplicate Saved View Name", "status:open", "events"); Assert.NotNull(created); + // Act & Assert await SendRequestAsync(r => r .Post() .AsGlobalAdminUser() @@ -269,9 +277,11 @@ await SendRequestAsync(r => r [Fact] public async Task PostAsync_DuplicateSlug_ReturnsConflict() { + // Arrange var created = await CreateSavedViewAsync("Shared URL Name", "status:open", "events", slug: "shared-url-name"); Assert.NotNull(created); + // Act & Assert await SendRequestAsync(r => r .Post() .AsGlobalAdminUser() @@ -291,6 +301,7 @@ await SendRequestAsync(r => r [Fact] public Task PostAsync_WithEmptyFilter_ReturnsCreated() { + // Arrange var newView = new NewSavedView { OrganizationId = SampleDataService.TEST_ORG_ID, @@ -299,6 +310,7 @@ public Task PostAsync_WithEmptyFilter_ReturnsCreated() ViewType = "events" }; + // Act & Assert return SendRequestAsync(r => r .Post() .AsGlobalAdminUser() @@ -311,6 +323,7 @@ public Task PostAsync_WithEmptyFilter_ReturnsCreated() [Fact] public Task PostAsync_WithInvalidView_ReturnsUnprocessableEntity() { + // Arrange var newView = new NewSavedView { OrganizationId = SampleDataService.TEST_ORG_ID, @@ -319,6 +332,7 @@ public Task PostAsync_WithInvalidView_ReturnsUnprocessableEntity() ViewType = "invalid-view" }; + // Act & Assert return SendRequestAsync(r => r .Post() .AsGlobalAdminUser() diff --git a/tests/Exceptionless.Tests/Exceptionless.Tests.csproj b/tests/Exceptionless.Tests/Exceptionless.Tests.csproj index 56d9f37f89..821d07f57e 100644 --- a/tests/Exceptionless.Tests/Exceptionless.Tests.csproj +++ b/tests/Exceptionless.Tests/Exceptionless.Tests.csproj @@ -8,7 +8,6 @@ - diff --git a/tests/Exceptionless.Tests/Extensions/JsonNodeExtensionsTests.cs b/tests/Exceptionless.Tests/Extensions/JsonNodeExtensionsTests.cs new file mode 100644 index 0000000000..c4b2d5ce51 --- /dev/null +++ b/tests/Exceptionless.Tests/Extensions/JsonNodeExtensionsTests.cs @@ -0,0 +1,118 @@ +using System.Text.Json.Nodes; +using Exceptionless.Core.Extensions; +using Xunit; + +namespace Exceptionless.Tests.Extensions; + +public class JsonNodeExtensionsTests : TestWithServices +{ + public JsonNodeExtensionsTests(ITestOutputHelper output) : base(output) { } + + [Fact] + public void Rename_WhenNewNameAlreadyExists_OverwritesWithRenamedValue() + { + // Arrange: "old_name" → "existing" but "existing" already has a value + var obj = new JsonObject + { + ["old_name"] = "renamed_value", + ["existing"] = "original_value", + ["other"] = "keep_me" + }; + + // Act: should not throw + bool result = obj.Rename("old_name", "existing"); + + // Assert: renamed value wins, old "existing" is overwritten + Assert.True(result); + Assert.False(obj.ContainsKey("old_name")); + Assert.Equal("renamed_value", obj["existing"]?.GetValue()); + Assert.Equal("keep_me", obj["other"]?.GetValue()); + } + + [Fact] + public void Rename_WhenNewNameDoesNotExist_RenamesNormally() + { + // Arrange + var obj = new JsonObject + { + ["old_name"] = "value", + ["other"] = "keep" + }; + + // Act + bool result = obj.Rename("old_name", "new_name"); + + // Assert + Assert.True(result); + Assert.False(obj.ContainsKey("old_name")); + Assert.Equal("value", obj["new_name"]?.GetValue()); + Assert.Equal("keep", obj["other"]?.GetValue()); + } + + [Fact] + public void Rename_SameNameNoOp_ReturnsTrue() + { + // Arrange + var obj = new JsonObject { ["name"] = "value" }; + + // Act + bool result = obj.Rename("name", "name"); + + // Assert + Assert.True(result); + Assert.Equal("value", obj["name"]?.GetValue()); + } + + [Fact] + public void Rename_PropertyNotFound_ReturnsFalse() + { + // Arrange + var obj = new JsonObject { ["other"] = "value" }; + + // Act + bool result = obj.Rename("missing", "new_name"); + + // Assert + Assert.False(result); + Assert.False(obj.ContainsKey("new_name")); + } + + [Fact] + public void RenameOrRemoveIfNullOrEmpty_WhenNewNameAlreadyExists_OverwritesWithRenamedValue() + { + // Arrange + var obj = new JsonObject + { + ["old_name"] = "renamed_value", + ["existing"] = "original_value" + }; + + // Act + bool result = obj.RenameOrRemoveIfNullOrEmpty("old_name", "existing"); + + // Assert + Assert.True(result); + Assert.False(obj.ContainsKey("old_name")); + Assert.Equal("renamed_value", obj["existing"]?.GetValue()); + } + + [Fact] + public void RenameOrRemoveIfNullOrEmpty_NullValue_RemovesProperty() + { + // Arrange + var obj = new JsonObject + { + ["old_name"] = null, + ["other"] = "keep" + }; + + // Act + bool result = obj.RenameOrRemoveIfNullOrEmpty("old_name", "new_name"); + + // Assert + Assert.False(result); + Assert.False(obj.ContainsKey("old_name")); + Assert.False(obj.ContainsKey("new_name")); + Assert.Equal("keep", obj["other"]?.GetValue()); + } +} diff --git a/tests/Exceptionless.Tests/IntegrationTestsBase.cs b/tests/Exceptionless.Tests/IntegrationTestsBase.cs index bcd8b4faa0..873c902192 100644 --- a/tests/Exceptionless.Tests/IntegrationTestsBase.cs +++ b/tests/Exceptionless.Tests/IntegrationTestsBase.cs @@ -1,4 +1,6 @@ using System.Text.Json; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.QueryDsl; using Exceptionless.Core.Authentication; using Exceptionless.Core.Extensions; using Exceptionless.Core.Mail; @@ -23,7 +25,6 @@ using Foundatio.Utility; using Foundatio.Xunit; using Microsoft.AspNetCore.TestHost; -using Nest; using Xunit; using HttpMethod = System.Net.Http.HttpMethod; using LogLevel = Microsoft.Extensions.Logging.LogLevel; @@ -165,7 +166,8 @@ await _configuration.Client.DeleteByQueryAsync(new DeleteByQueryRequest(indexes) { Query = new MatchAllQuery(), IgnoreUnavailable = true, - Refresh = true + Refresh = true, + Conflicts = Conflicts.Proceed }); } diff --git a/tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs b/tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs index 091ca285de..24deba17f9 100644 --- a/tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs +++ b/tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs @@ -1,6 +1,7 @@ using Exceptionless.Core; using Exceptionless.Core.Billing; using Exceptionless.Core.Jobs; +using Exceptionless.Core.Models; using Exceptionless.Core.Repositories; using Exceptionless.DateTimeExtensions; using Exceptionless.Tests.Utility; @@ -54,7 +55,7 @@ public async Task CanCleanupSuspendedTokens() organization.SuspensionNotes = "blah"; await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); - var project = await _projectRepository.AddAsync(_projectData.GenerateSampleProject()); + await _projectRepository.AddAsync(_projectData.GenerateSampleProject()); var token = await _tokenRepository.AddAsync(_tokenData.GenerateSampleApiKeyToken(), o => o.ImmediateConsistency()); Assert.False(token.IsSuspended); @@ -136,7 +137,7 @@ public async Task CanCleanupEventsOutsideOfRetentionPeriod() var options = GetService(); var date = DateTimeOffset.UtcNow.SubtractDays(options.MaximumRetentionDays); - var persistentEvent = await _eventRepository.AddAsync(_eventData.GenerateEvent(organization.Id, project.Id, stack.Id, date, date, date), o => o.ImmediateConsistency()); + var persistentEvent = await _eventRepository.AddAsync(_eventData.GenerateEvent(organization.Id, project.Id, stack.Id, occurrenceDate: date), o => o.ImmediateConsistency()); await _job.RunAsync(TestCancellationToken); @@ -169,4 +170,400 @@ public async Task CanDeleteOrphanedEventsByStack() eventCount = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); Assert.Equal(5000, eventCount); } + + [Fact] + public async Task CanCleanupSuspendedTokens_MultiTenant_OnlySuspendedOrganizationTokensAffected() + { + // Arrange - Organization 1 is suspended, Organization 2 is active + var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); + organization1.IsSuspended = true; + organization1.SuspensionDate = DateTime.UtcNow; + organization1.SuspendedByUserId = TestConstants.UserId; + organization1.SuspensionCode = Core.Models.SuspensionCode.Billing; + await _organizationRepository.AddAsync(organization1, o => o.ImmediateConsistency()); + + var organization2 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId2); + await _organizationRepository.AddAsync(organization2, o => o.ImmediateConsistency()); + + // Tokens for both organizations + var token1 = _tokenData.GenerateToken(generateId: true, organizationId: organization1.Id, projectId: TestConstants.ProjectId); + var token2 = _tokenData.GenerateToken(generateId: true, organizationId: organization2.Id, projectId: TestConstants.ProjectIdWithNoRoles); + await _tokenRepository.AddAsync([token1, token2], o => o.ImmediateConsistency()); + + Assert.False(token1.IsSuspended); + Assert.False(token2.IsSuspended); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert - Only Organization 1's token is suspended + var updatedToken1 = await _tokenRepository.GetByIdAsync(token1.Id); + var updatedToken2 = await _tokenRepository.GetByIdAsync(token2.Id); + Assert.NotNull(updatedToken1); + Assert.NotNull(updatedToken2); + Assert.True(updatedToken1.IsSuspended); + Assert.False(updatedToken2.IsSuspended); + } + + [Fact] + public async Task CanCleanupSuspendedTokens_AlreadySuspendedToken_RemainsUnchanged() + { + // Arrange - Organization suspended, token already marked as suspended + var organization = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); + organization.IsSuspended = true; + organization.SuspensionDate = DateTime.UtcNow; + organization.SuspendedByUserId = TestConstants.UserId; + organization.SuspensionCode = Core.Models.SuspensionCode.Abuse; + await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); + + var token = _tokenData.GenerateToken(generateId: true, organizationId: organization.Id, projectId: TestConstants.ProjectId); + token.IsSuspended = true; + await _tokenRepository.AddAsync(token, o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert - Token remains suspended + var updatedToken = await _tokenRepository.GetByIdAsync(token.Id); + Assert.NotNull(updatedToken); + Assert.True(updatedToken.IsSuspended); + } + + [Fact] + public async Task CanCleanupSuspendedTokens_MultipleTokensPerOrganization_AllGetSuspended() + { + // Arrange - Suspended organization with many tokens + var organization = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); + organization.IsSuspended = true; + organization.SuspensionDate = DateTime.UtcNow; + organization.SuspendedByUserId = TestConstants.UserId; + organization.SuspensionCode = Core.Models.SuspensionCode.Billing; + await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); + + var tokens = new List(); + for (int i = 0; i < 10; i++) + tokens.Add(_tokenData.GenerateToken(generateId: true, organizationId: organization.Id, projectId: TestConstants.ProjectId)); + await _tokenRepository.AddAsync(tokens, o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert - All 10 tokens suspended + foreach (var t in tokens) + { + var updated = await _tokenRepository.GetByIdAsync(t.Id); + Assert.NotNull(updated); + Assert.True(updated.IsSuspended); + } + } + + [Fact] + public async Task CanCleanupSuspendedTokens_NoSuspendedOrganizations_NoTokensModified() + { + // Arrange - Two active organizations with tokens + var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); + var organization2 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId2); + await _organizationRepository.AddAsync([organization1, organization2], o => o.ImmediateConsistency()); + + var token1 = _tokenData.GenerateToken(generateId: true, organizationId: organization1.Id, projectId: TestConstants.ProjectId); + var token2 = _tokenData.GenerateToken(generateId: true, organizationId: organization2.Id, projectId: TestConstants.ProjectIdWithNoRoles); + await _tokenRepository.AddAsync([token1, token2], o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert - No tokens suspended + var updated1 = await _tokenRepository.GetByIdAsync(token1.Id); + var updated2 = await _tokenRepository.GetByIdAsync(token2.Id); + Assert.NotNull(updated1); + Assert.NotNull(updated2); + Assert.False(updated1.IsSuspended); + Assert.False(updated2.IsSuspended); + } + + [Fact] + public async Task CanCleanupSoftDeletedOrganization_MultiTenant_OnlyDeletedOrganizationCleaned() + { + // Arrange - Organization 1 is soft-deleted, Organization 2 is active + var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); + organization1.IsDeleted = true; + await _organizationRepository.AddAsync(organization1, o => o.ImmediateConsistency()); + + var organization2 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId2); + await _organizationRepository.AddAsync(organization2, o => o.ImmediateConsistency()); + + var project1 = await _projectRepository.AddAsync(_projectData.GenerateProject(id: TestConstants.ProjectId, organizationId: organization1.Id), o => o.ImmediateConsistency()); + var project2 = await _projectRepository.AddAsync(_projectData.GenerateProject(generateId: true, organizationId: organization2.Id), o => o.ImmediateConsistency()); + + var stack1 = await _stackRepository.AddAsync(_stackData.GenerateStack(id: TestConstants.StackId, organizationId: organization1.Id, projectId: project1.Id), o => o.ImmediateConsistency()); + var stack2 = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: organization2.Id, projectId: project2.Id), o => o.ImmediateConsistency()); + + var event1 = await _eventRepository.AddAsync(_eventData.GenerateEvent(organization1.Id, project1.Id, stack1.Id), o => o.ImmediateConsistency()); + var event2 = await _eventRepository.AddAsync(_eventData.GenerateEvent(organization2.Id, project2.Id, stack2.Id), o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert - Organization 1's entire hierarchy is hard-deleted; Organization 2's everything remains + Assert.Null(await _organizationRepository.GetByIdAsync(organization1.Id, o => o.IncludeSoftDeletes())); + Assert.Null(await _projectRepository.GetByIdAsync(project1.Id, o => o.IncludeSoftDeletes())); + Assert.Null(await _stackRepository.GetByIdAsync(stack1.Id, o => o.IncludeSoftDeletes())); + Assert.Null(await _eventRepository.GetByIdAsync(event1.Id, o => o.IncludeSoftDeletes())); + + Assert.NotNull(await _organizationRepository.GetByIdAsync(organization2.Id)); + Assert.NotNull(await _projectRepository.GetByIdAsync(project2.Id)); + Assert.NotNull(await _stackRepository.GetByIdAsync(stack2.Id)); + Assert.NotNull(await _eventRepository.GetByIdAsync(event2.Id)); + } + + [Fact] + public async Task CanCleanupSoftDeletedProject_MultiTenant_OnlyDeletedProjectCleaned() + { + // Arrange - Two organizations, Organization 1 project is soft-deleted, Organization 2 project is active + var organization1 = await _organizationRepository.AddAsync(_organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId), o => o.ImmediateConsistency()); + var organization2 = await _organizationRepository.AddAsync(_organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId2), o => o.ImmediateConsistency()); + + var project1 = _projectData.GenerateProject(id: TestConstants.ProjectId, organizationId: organization1.Id); + project1.IsDeleted = true; + await _projectRepository.AddAsync(project1, o => o.ImmediateConsistency()); + + var project2 = await _projectRepository.AddAsync(_projectData.GenerateProject(generateId: true, organizationId: organization2.Id), o => o.ImmediateConsistency()); + + var stack1 = await _stackRepository.AddAsync(_stackData.GenerateStack(id: TestConstants.StackId, organizationId: organization1.Id, projectId: project1.Id), o => o.ImmediateConsistency()); + var stack2 = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: organization2.Id, projectId: project2.Id), o => o.ImmediateConsistency()); + + var event1 = await _eventRepository.AddAsync(_eventData.GenerateEvent(organization1.Id, project1.Id, stack1.Id), o => o.ImmediateConsistency()); + var event2 = await _eventRepository.AddAsync(_eventData.GenerateEvent(organization2.Id, project2.Id, stack2.Id), o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert - Project 1's stacks/events gone; Project 2 and both organizations remain + Assert.NotNull(await _organizationRepository.GetByIdAsync(organization1.Id)); + Assert.NotNull(await _organizationRepository.GetByIdAsync(organization2.Id)); + Assert.Null(await _projectRepository.GetByIdAsync(project1.Id, o => o.IncludeSoftDeletes())); + Assert.NotNull(await _projectRepository.GetByIdAsync(project2.Id)); + Assert.Null(await _stackRepository.GetByIdAsync(stack1.Id, o => o.IncludeSoftDeletes())); + Assert.NotNull(await _stackRepository.GetByIdAsync(stack2.Id)); + Assert.Null(await _eventRepository.GetByIdAsync(event1.Id, o => o.IncludeSoftDeletes())); + Assert.NotNull(await _eventRepository.GetByIdAsync(event2.Id)); + } + + [Fact] + public async Task CanCleanupSoftDeletedStack_MultiTenant_OnlyDeletedStackCleaned() + { + // Arrange - Same organization, two projects, one stack soft-deleted in project 1 + var organization = await _organizationRepository.AddAsync(_organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId), o => o.ImmediateConsistency()); + var project = await _projectRepository.AddAsync(_projectData.GenerateSampleProject(), o => o.ImmediateConsistency()); + + var stack1 = _stackData.GenerateStack(id: TestConstants.StackId, organizationId: organization.Id, projectId: project.Id); + stack1.IsDeleted = true; + await _stackRepository.AddAsync(stack1, o => o.ImmediateConsistency()); + + var stack2 = _stackData.GenerateStack(generateId: true, organizationId: organization.Id, projectId: project.Id); + await _stackRepository.AddAsync(stack2, o => o.ImmediateConsistency()); + + var event1 = await _eventRepository.AddAsync(_eventData.GenerateEvent(organization.Id, project.Id, stack1.Id), o => o.ImmediateConsistency()); + var event2 = await _eventRepository.AddAsync(_eventData.GenerateEvent(organization.Id, project.Id, stack2.Id), o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert + Assert.NotNull(await _organizationRepository.GetByIdAsync(organization.Id)); + Assert.NotNull(await _projectRepository.GetByIdAsync(project.Id)); + Assert.Null(await _stackRepository.GetByIdAsync(stack1.Id, o => o.IncludeSoftDeletes())); + Assert.NotNull(await _stackRepository.GetByIdAsync(stack2.Id)); + Assert.Null(await _eventRepository.GetByIdAsync(event1.Id, o => o.IncludeSoftDeletes())); + Assert.NotNull(await _eventRepository.GetByIdAsync(event2.Id)); + } + + [Fact] + public async Task CanCleanupEventsOutsideOfRetentionPeriod_MultiTenant_OnlyExpiredEventsRemoved() + { + // Arrange - Two organizations on free plan, each has events inside and outside retention + var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); + _billingManager.ApplyBillingPlan(organization1, _plans.FreePlan); + var organization2 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId2); + _billingManager.ApplyBillingPlan(organization2, _plans.FreePlan); + await _organizationRepository.AddAsync([organization1, organization2], o => o.ImmediateConsistency()); + + var project1 = await _projectRepository.AddAsync(_projectData.GenerateProject(id: TestConstants.ProjectId, organizationId: organization1.Id), o => o.ImmediateConsistency()); + var project2 = await _projectRepository.AddAsync(_projectData.GenerateProject(generateId: true, organizationId: organization2.Id), o => o.ImmediateConsistency()); + + var stack1 = await _stackRepository.AddAsync(_stackData.GenerateStack(id: TestConstants.StackId, organizationId: organization1.Id, projectId: project1.Id), o => o.ImmediateConsistency()); + var stack2 = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: organization2.Id, projectId: project2.Id), o => o.ImmediateConsistency()); + + var options = GetService(); + + // Create "will-be-expired" events at a date that's valid for index insertion + // then advance time so they fall outside retention + var willExpireDate = DateTimeOffset.UtcNow.SubtractDays(options.MaximumRetentionDays); + var recentDate = DateTimeOffset.UtcNow.AddDays(-1); + + // Organization 1: 1 recent (keep) + 1 at retention boundary (will be expired after time advance) + var recentEvent1 = await _eventRepository.AddAsync(_eventData.GenerateEvent(organization1.Id, project1.Id, stack1.Id, occurrenceDate: recentDate), o => o.ImmediateConsistency()); + var expiredEvent1 = await _eventRepository.AddAsync(_eventData.GenerateEvent(organization1.Id, project1.Id, stack1.Id, occurrenceDate: willExpireDate), o => o.ImmediateConsistency()); + + // Organization 2: 1 recent (keep) + 1 at retention boundary + var recentEvent2 = await _eventRepository.AddAsync(_eventData.GenerateEvent(organization2.Id, project2.Id, stack2.Id, occurrenceDate: recentDate), o => o.ImmediateConsistency()); + var expiredEvent2 = await _eventRepository.AddAsync(_eventData.GenerateEvent(organization2.Id, project2.Id, stack2.Id, occurrenceDate: willExpireDate), o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert - Only recent events survive (events at retention boundary are deleted) + Assert.NotNull(await _eventRepository.GetByIdAsync(recentEvent1.Id)); + Assert.NotNull(await _eventRepository.GetByIdAsync(recentEvent2.Id)); + Assert.Null(await _eventRepository.GetByIdAsync(expiredEvent1.Id, o => o.IncludeSoftDeletes())); + Assert.Null(await _eventRepository.GetByIdAsync(expiredEvent2.Id, o => o.IncludeSoftDeletes())); + } + + [Fact] + public async Task CanCleanupEventsOutsideOfRetentionPeriod_PaidPlan_HasLongerRetention() + { + // Arrange - Organization 1 on free plan (short retention), Organization 2 on unlimited (long retention) + var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); + _billingManager.ApplyBillingPlan(organization1, _plans.FreePlan); + var organization2 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId2); + _billingManager.ApplyBillingPlan(organization2, _plans.UnlimitedPlan); + await _organizationRepository.AddAsync([organization1, organization2], o => o.ImmediateConsistency()); + + var project1 = await _projectRepository.AddAsync(_projectData.GenerateProject(id: TestConstants.ProjectId, organizationId: organization1.Id), o => o.ImmediateConsistency()); + var project2 = await _projectRepository.AddAsync(_projectData.GenerateProject(generateId: true, organizationId: organization2.Id), o => o.ImmediateConsistency()); + + var stack1 = await _stackRepository.AddAsync(_stackData.GenerateStack(id: TestConstants.StackId, organizationId: organization1.Id, projectId: project1.Id), o => o.ImmediateConsistency()); + var stack2 = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: organization2.Id, projectId: project2.Id), o => o.ImmediateConsistency()); + + // Both events at the free plan retention boundary + var options = GetService(); + var dateAtFreeRetentionBoundary = DateTimeOffset.UtcNow.SubtractDays(options.MaximumRetentionDays); + + var event1 = await _eventRepository.AddAsync(_eventData.GenerateEvent(organization1.Id, project1.Id, stack1.Id, occurrenceDate: dateAtFreeRetentionBoundary), o => o.ImmediateConsistency()); + var event2 = await _eventRepository.AddAsync(_eventData.GenerateEvent(organization2.Id, project2.Id, stack2.Id, occurrenceDate: dateAtFreeRetentionBoundary), o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert - Free plan event deleted; unlimited plan event preserved + Assert.Null(await _eventRepository.GetByIdAsync(event1.Id, o => o.IncludeSoftDeletes())); + Assert.NotNull(await _eventRepository.GetByIdAsync(event2.Id)); + } + + [Fact] + public async Task FullCleanup_ComplexMultiTenantScenario_AllRulesAppliedCorrectly() + { + // Arrange - Complex scenario: suspended organization, soft-deleted project, soft-deleted stack, retention + // Organization 1: suspended (tokens get suspended) + var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); + organization1.IsSuspended = true; + organization1.SuspensionDate = DateTime.UtcNow; + organization1.SuspendedByUserId = TestConstants.UserId; + organization1.SuspensionCode = Core.Models.SuspensionCode.Billing; + await _organizationRepository.AddAsync(organization1, o => o.ImmediateConsistency()); + + // Organization 2: has a soft-deleted project (project and children get cleaned up) + var organization2 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId2); + await _organizationRepository.AddAsync(organization2, o => o.ImmediateConsistency()); + + // Organization 1 token (should become suspended) + var token1 = _tokenData.GenerateToken(generateId: true, organizationId: organization1.Id, projectId: TestConstants.ProjectId); + await _tokenRepository.AddAsync(token1, o => o.ImmediateConsistency()); + + // Organization 2 token (should remain unsuspended) + var token2 = _tokenData.GenerateToken(generateId: true, organizationId: organization2.Id, projectId: TestConstants.ProjectIdWithNoRoles); + await _tokenRepository.AddAsync(token2, o => o.ImmediateConsistency()); + + // Project 1 for Organization 1 (active project, suspended organization) + var project1 = await _projectRepository.AddAsync(_projectData.GenerateProject(id: TestConstants.ProjectId, organizationId: organization1.Id), o => o.ImmediateConsistency()); + + // Project 2 for Organization 2 (soft-deleted project) + var project2 = _projectData.GenerateProject(generateId: true, organizationId: organization2.Id); + project2.IsDeleted = true; + await _projectRepository.AddAsync(project2, o => o.ImmediateConsistency()); + + // Project 3 for Organization 2 (active project) + var project3 = await _projectRepository.AddAsync(_projectData.GenerateProject(generateId: true, organizationId: organization2.Id), o => o.ImmediateConsistency()); + + // Stacks and events + var stack1 = await _stackRepository.AddAsync(_stackData.GenerateStack(id: TestConstants.StackId, organizationId: organization1.Id, projectId: project1.Id), o => o.ImmediateConsistency()); + var stack2 = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: organization2.Id, projectId: project2.Id), o => o.ImmediateConsistency()); + var stack3 = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: organization2.Id, projectId: project3.Id), o => o.ImmediateConsistency()); + + var event1 = await _eventRepository.AddAsync(_eventData.GenerateEvent(organization1.Id, project1.Id, stack1.Id), o => o.ImmediateConsistency()); + var event2 = await _eventRepository.AddAsync(_eventData.GenerateEvent(organization2.Id, project2.Id, stack2.Id), o => o.ImmediateConsistency()); + var event3 = await _eventRepository.AddAsync(_eventData.GenerateEvent(organization2.Id, project3.Id, stack3.Id), o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert + // Token 1 suspended (organization is suspended) + var updatedToken1 = await _tokenRepository.GetByIdAsync(token1.Id); + Assert.NotNull(updatedToken1); + Assert.True(updatedToken1.IsSuspended); + + // Token 2 not suspended (organization is active) + var updatedToken2 = await _tokenRepository.GetByIdAsync(token2.Id); + Assert.NotNull(updatedToken2); + Assert.False(updatedToken2.IsSuspended); + + // Organization 1 exists (suspended != deleted), its project/stack/event remain + Assert.NotNull(await _organizationRepository.GetByIdAsync(organization1.Id)); + Assert.NotNull(await _projectRepository.GetByIdAsync(project1.Id)); + Assert.NotNull(await _stackRepository.GetByIdAsync(stack1.Id)); + Assert.NotNull(await _eventRepository.GetByIdAsync(event1.Id)); + + // Organization 2 exists, soft-deleted project2 is cleaned up + Assert.NotNull(await _organizationRepository.GetByIdAsync(organization2.Id)); + Assert.Null(await _projectRepository.GetByIdAsync(project2.Id, o => o.IncludeSoftDeletes())); + Assert.Null(await _stackRepository.GetByIdAsync(stack2.Id, o => o.IncludeSoftDeletes())); + Assert.Null(await _eventRepository.GetByIdAsync(event2.Id, o => o.IncludeSoftDeletes())); + + // Organization 2's active project3 untouched + Assert.NotNull(await _projectRepository.GetByIdAsync(project3.Id)); + Assert.NotNull(await _stackRepository.GetByIdAsync(stack3.Id)); + Assert.NotNull(await _eventRepository.GetByIdAsync(event3.Id)); + } + + [Fact] + public async Task FullCleanup_NoDataToClean_CompletesSuccessfully() + { + // Arrange - Two healthy active organizations, no soft deletes, no suspended, no expired + var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); + var organization2 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId2); + await _organizationRepository.AddAsync([organization1, organization2], o => o.ImmediateConsistency()); + + var project1 = await _projectRepository.AddAsync(_projectData.GenerateProject(id: TestConstants.ProjectId, organizationId: organization1.Id), o => o.ImmediateConsistency()); + var project2 = await _projectRepository.AddAsync(_projectData.GenerateProject(generateId: true, organizationId: organization2.Id), o => o.ImmediateConsistency()); + + var stack1 = await _stackRepository.AddAsync(_stackData.GenerateStack(id: TestConstants.StackId, organizationId: organization1.Id, projectId: project1.Id), o => o.ImmediateConsistency()); + var stack2 = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: organization2.Id, projectId: project2.Id), o => o.ImmediateConsistency()); + + await _eventRepository.AddAsync(_eventData.GenerateEvents(10, organization1.Id, project1.Id, stack1.Id), o => o.ImmediateConsistency()); + await _eventRepository.AddAsync(_eventData.GenerateEvents(10, organization2.Id, project2.Id, stack2.Id), o => o.ImmediateConsistency()); + + var token1 = _tokenData.GenerateToken(generateId: true, organizationId: organization1.Id, projectId: project1.Id); + var token2 = _tokenData.GenerateToken(generateId: true, organizationId: organization2.Id, projectId: project2.Id); + await _tokenRepository.AddAsync([token1, token2], o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert - Everything remains + Assert.NotNull(await _organizationRepository.GetByIdAsync(organization1.Id)); + Assert.NotNull(await _organizationRepository.GetByIdAsync(organization2.Id)); + Assert.NotNull(await _projectRepository.GetByIdAsync(project1.Id)); + Assert.NotNull(await _projectRepository.GetByIdAsync(project2.Id)); + Assert.NotNull(await _stackRepository.GetByIdAsync(stack1.Id)); + Assert.NotNull(await _stackRepository.GetByIdAsync(stack2.Id)); + + var eventCount = await _eventRepository.CountAsync(o => o.ImmediateConsistency()); + Assert.Equal(20, eventCount); + + var updatedToken1 = await _tokenRepository.GetByIdAsync(token1.Id); + var updatedToken2 = await _tokenRepository.GetByIdAsync(token2.Id); + Assert.False(updatedToken1!.IsSuspended); + Assert.False(updatedToken2!.IsSuspended); + } } diff --git a/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobTests.cs b/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobTests.cs new file mode 100644 index 0000000000..af3f405b16 --- /dev/null +++ b/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobTests.cs @@ -0,0 +1,549 @@ +using Exceptionless.Core; +using Exceptionless.Core.Billing; +using Exceptionless.Core.Jobs; +using Exceptionless.Core.Models; +using Exceptionless.Core.Repositories; +using Exceptionless.Tests.Utility; +using Foundatio.Repositories; +using Foundatio.Repositories.Utility; +using Xunit; + +namespace Exceptionless.Tests.Jobs; + +public class CleanupOrphanedDataJobTests : IntegrationTestsBase +{ + private readonly CleanupOrphanedDataJob _job; + private readonly OrganizationData _organizationData; + private readonly IOrganizationRepository _organizationRepository; + private readonly ProjectData _projectData; + private readonly IProjectRepository _projectRepository; + private readonly StackData _stackData; + private readonly IStackRepository _stackRepository; + private readonly EventData _eventData; + private readonly IEventRepository _eventRepository; + private readonly BillingManager _billingManager; + private readonly BillingPlans _plans; + + public CleanupOrphanedDataJobTests(ITestOutputHelper output, AppWebHostFactory factory) : base(output, factory) + { + _job = GetService(); + _organizationData = GetService(); + _organizationRepository = GetService(); + _projectData = GetService(); + _projectRepository = GetService(); + _stackData = GetService(); + _stackRepository = GetService(); + _eventData = GetService(); + _eventRepository = GetService(); + _billingManager = GetService(); + _plans = GetService(); + } + + [Fact] + public async Task DeleteOrphanedEventsByStack_WithValidStack_DoesNotDeleteEvents() + { + // Arrange - Two tenants, each with valid stacks and events + var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); + var organization2 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId2); + await _organizationRepository.AddAsync([organization1, organization2], o => o.ImmediateConsistency()); + + var project1 = _projectData.GenerateProject(id: TestConstants.ProjectId, organizationId: organization1.Id); + var project2 = _projectData.GenerateProject(generateId: true, organizationId: organization2.Id); + await _projectRepository.AddAsync([project1, project2], o => o.ImmediateConsistency()); + + var stack1 = _stackData.GenerateStack(id: TestConstants.StackId, organizationId: organization1.Id, projectId: project1.Id); + var stack2 = _stackData.GenerateStack(generateId: true, organizationId: organization2.Id, projectId: project2.Id); + await _stackRepository.AddAsync([stack1, stack2], o => o.ImmediateConsistency()); + + var events1 = _eventData.GenerateEvents(100, organization1.Id, project1.Id, stack1.Id).ToList(); + var events2 = _eventData.GenerateEvents(100, organization2.Id, project2.Id, stack2.Id).ToList(); + await _eventRepository.AddAsync(events1.Concat(events2), o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert - All events should remain (no orphans) + var totalCount = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); + Assert.Equal(200, totalCount); + } + + [Fact] + public async Task DeleteOrphanedEventsByStack_WithMixedOrphanedAndValid_OnlyDeletesOrphaned() + { + // Arrange - Tenant 1 has valid events; Tenant 2 has orphaned events (stack doesn't exist) + var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); + var organization2 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId2); + await _organizationRepository.AddAsync([organization1, organization2], o => o.ImmediateConsistency()); + + var project1 = _projectData.GenerateProject(id: TestConstants.ProjectId, organizationId: organization1.Id); + var project2 = _projectData.GenerateProject(generateId: true, organizationId: organization2.Id); + await _projectRepository.AddAsync([project1, project2], o => o.ImmediateConsistency()); + + var stack1 = _stackData.GenerateStack(id: TestConstants.StackId, organizationId: organization1.Id, projectId: project1.Id); + var stack2 = _stackData.GenerateStack(generateId: true, organizationId: organization2.Id, projectId: project2.Id); + await _stackRepository.AddAsync([stack1, stack2], o => o.ImmediateConsistency()); + + // Valid events for both tenants + var validEvents1 = _eventData.GenerateEvents(50, organization1.Id, project1.Id, stack1.Id).ToList(); + var validEvents2 = _eventData.GenerateEvents(50, organization2.Id, project2.Id, stack2.Id).ToList(); + + // Orphaned events (stack IDs that don't exist) in both tenants + string fakeStackId1 = ObjectId.GenerateNewId().ToString(); + string fakeStackId2 = ObjectId.GenerateNewId().ToString(); + var orphanedEvents1 = _eventData.GenerateEvents(30, organization1.Id, project1.Id, fakeStackId1).ToList(); + var orphanedEvents2 = _eventData.GenerateEvents(20, organization2.Id, project2.Id, fakeStackId2).ToList(); + + await _eventRepository.AddAsync(validEvents1.Concat(validEvents2).Concat(orphanedEvents1).Concat(orphanedEvents2), o => o.ImmediateConsistency()); + + var totalBefore = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); + Assert.Equal(150, totalBefore); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert - Only valid events remain + var totalAfter = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); + Assert.Equal(100, totalAfter); + } + + [Fact] + public async Task DeleteOrphanedEventsByStack_LargeVolume_PreservesAllValidEvents() + { + // Arrange - Large volume across two tenants: 5000 valid + 10000 orphaned + var organization = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); + await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); + + var project = _projectData.GenerateProject(id: TestConstants.ProjectId, organizationId: organization.Id); + await _projectRepository.AddAsync(project, o => o.ImmediateConsistency()); + + var stack = _stackData.GenerateStack(id: TestConstants.StackId, organizationId: organization.Id, projectId: project.Id); + await _stackRepository.AddAsync(stack, o => o.ImmediateConsistency()); + + // 5000 valid events for existing stack + await _eventRepository.AddAsync(_eventData.GenerateEvents(5000, organization.Id, project.Id, stack.Id), o => o.ImmediateConsistency()); + + // 10000 orphaned events with many different fake stack IDs + var orphanedEvents = _eventData.GenerateEvents(10000, organization.Id, project.Id).ToList(); + orphanedEvents.ForEach(e => e.StackId = ObjectId.GenerateNewId().ToString()); + await _eventRepository.AddAsync(orphanedEvents, o => o.ImmediateConsistency()); + + var totalBefore = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); + Assert.Equal(15000, totalBefore); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert - Only the 5000 valid events remain + var totalAfter = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); + Assert.Equal(5000, totalAfter); + } + + [Fact] + public async Task DeleteOrphanedEventsByStack_MultipleValidStacks_PreservesAll() + { + // Arrange - Multiple valid stacks in two organizations, no orphans + var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); + var organization2 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId2); + await _organizationRepository.AddAsync([organization1, organization2], o => o.ImmediateConsistency()); + + var project1 = _projectData.GenerateProject(id: TestConstants.ProjectId, organizationId: organization1.Id); + var project2 = _projectData.GenerateProject(generateId: true, organizationId: organization2.Id); + await _projectRepository.AddAsync([project1, project2], o => o.ImmediateConsistency()); + + // Multiple stacks per project + var stacks = new List(); + for (int i = 0; i < 10; i++) + { + stacks.Add(_stackData.GenerateStack(generateId: true, organizationId: organization1.Id, projectId: project1.Id)); + stacks.Add(_stackData.GenerateStack(generateId: true, organizationId: organization2.Id, projectId: project2.Id)); + } + await _stackRepository.AddAsync(stacks, o => o.ImmediateConsistency()); + + // Events across all stacks + var events = new List(); + foreach (var stack in stacks) + events.AddRange(_eventData.GenerateEvents(10, stack.OrganizationId, stack.ProjectId, stack.Id)); + await _eventRepository.AddAsync(events, o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert - All 200 events preserved + var totalAfter = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); + Assert.Equal(200, totalAfter); + } + + [Fact] + public async Task DeleteOrphanedEventsByStack_OnlyOrphanedEventsInOneTenant_OtherTenantUnaffected() + { + // Arrange - Tenant 1 has all orphaned events (will be deleted); Tenant 2 has all valid events + var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); + var organization2 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId2); + await _organizationRepository.AddAsync([organization1, organization2], o => o.ImmediateConsistency()); + + var project1 = _projectData.GenerateProject(id: TestConstants.ProjectId, organizationId: organization1.Id); + var project2 = _projectData.GenerateProject(generateId: true, organizationId: organization2.Id); + await _projectRepository.AddAsync([project1, project2], o => o.ImmediateConsistency()); + + // Tenant 2 has a valid stack + var validStack = _stackData.GenerateStack(generateId: true, organizationId: organization2.Id, projectId: project2.Id); + await _stackRepository.AddAsync(validStack, o => o.ImmediateConsistency()); + + // Tenant 1 events are all orphaned (fake stack IDs) + var orphanedEvents = _eventData.GenerateEvents(100, organization1.Id, project1.Id).ToList(); + orphanedEvents.ForEach(e => e.StackId = ObjectId.GenerateNewId().ToString()); + + // Tenant 2 events are all valid + var validEvents = _eventData.GenerateEvents(100, organization2.Id, project2.Id, validStack.Id).ToList(); + + await _eventRepository.AddAsync(orphanedEvents.Concat(validEvents), o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert - Only tenant 2's events remain + var totalAfter = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); + Assert.Equal(100, totalAfter); + } + + [Fact] + public async Task DeleteOrphanedEventsByProject_WithValidProjects_DoesNotDeleteEvents() + { + // Arrange + var organization = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); + await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); + + var project1 = _projectData.GenerateProject(id: TestConstants.ProjectId, organizationId: organization.Id); + var project2 = _projectData.GenerateProject(generateId: true, organizationId: organization.Id); + await _projectRepository.AddAsync([project1, project2], o => o.ImmediateConsistency()); + + var stack1 = _stackData.GenerateStack(id: TestConstants.StackId, organizationId: organization.Id, projectId: project1.Id); + var stack2 = _stackData.GenerateStack(generateId: true, organizationId: organization.Id, projectId: project2.Id); + await _stackRepository.AddAsync([stack1, stack2], o => o.ImmediateConsistency()); + + var events = _eventData.GenerateEvents(50, organization.Id, project1.Id, stack1.Id) + .Concat(_eventData.GenerateEvents(50, organization.Id, project2.Id, stack2.Id)).ToList(); + await _eventRepository.AddAsync(events, o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert + var totalAfter = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); + Assert.Equal(100, totalAfter); + } + + [Fact] + public async Task DeleteOrphanedEventsByProject_WithOrphanedProject_DeletesEventsForMissingProject() + { + // Arrange - Events reference a project that doesn't exist + var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); + var organization2 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId2); + await _organizationRepository.AddAsync([organization1, organization2], o => o.ImmediateConsistency()); + + var validProject = _projectData.GenerateProject(id: TestConstants.ProjectId, organizationId: organization1.Id); + await _projectRepository.AddAsync(validProject, o => o.ImmediateConsistency()); + + var validStack = _stackData.GenerateStack(id: TestConstants.StackId, organizationId: organization1.Id, projectId: validProject.Id); + await _stackRepository.AddAsync(validStack, o => o.ImmediateConsistency()); + + // Valid events for existing project + var validEvents = _eventData.GenerateEvents(75, organization1.Id, validProject.Id, validStack.Id).ToList(); + + // Orphaned events referencing a non-existent project in organization 2 + string fakeProjectId = ObjectId.GenerateNewId().ToString(); + string fakeStackId = ObjectId.GenerateNewId().ToString(); + var orphanedEvents = _eventData.GenerateEvents(50, organization2.Id, fakeProjectId, fakeStackId).ToList(); + + await _eventRepository.AddAsync(validEvents.Concat(orphanedEvents), o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert - Orphaned events deleted, valid events preserved + var totalAfter = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); + Assert.Equal(75, totalAfter); + } + + [Fact] + public async Task DeleteOrphanedEventsByProject_MultiTenant_EachTenantIndependent() + { + // Arrange - Tenant 1 has valid project, Tenant 2 has orphaned project + var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); + var organization2 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId2); + await _organizationRepository.AddAsync([organization1, organization2], o => o.ImmediateConsistency()); + + // Only Tenant 1 has a real project + var project1 = _projectData.GenerateProject(id: TestConstants.ProjectId, organizationId: organization1.Id); + await _projectRepository.AddAsync(project1, o => o.ImmediateConsistency()); + + var stack1 = _stackData.GenerateStack(id: TestConstants.StackId, organizationId: organization1.Id, projectId: project1.Id); + await _stackRepository.AddAsync(stack1, o => o.ImmediateConsistency()); + + // Tenant 1 valid events + var validEvents = _eventData.GenerateEvents(60, organization1.Id, project1.Id, stack1.Id).ToList(); + + // Tenant 2 orphaned events (project doesn't exist) + string nonExistentProjectId = ObjectId.GenerateNewId().ToString(); + string fakeStackId = ObjectId.GenerateNewId().ToString(); + var orphanedEvents = _eventData.GenerateEvents(40, organization2.Id, nonExistentProjectId, fakeStackId).ToList(); + + await _eventRepository.AddAsync(validEvents.Concat(orphanedEvents), o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert + var totalAfter = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); + Assert.Equal(60, totalAfter); + } + + [Fact] + public async Task DeleteOrphanedEventsByOrganization_WithValidOrganizations_DoesNotDeleteEvents() + { + // Arrange + var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); + var organization2 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId2); + await _organizationRepository.AddAsync([organization1, organization2], o => o.ImmediateConsistency()); + + var project1 = _projectData.GenerateProject(id: TestConstants.ProjectId, organizationId: organization1.Id); + var project2 = _projectData.GenerateProject(generateId: true, organizationId: organization2.Id); + await _projectRepository.AddAsync([project1, project2], o => o.ImmediateConsistency()); + + var stack1 = _stackData.GenerateStack(id: TestConstants.StackId, organizationId: organization1.Id, projectId: project1.Id); + var stack2 = _stackData.GenerateStack(generateId: true, organizationId: organization2.Id, projectId: project2.Id); + await _stackRepository.AddAsync([stack1, stack2], o => o.ImmediateConsistency()); + + var events = _eventData.GenerateEvents(80, organization1.Id, project1.Id, stack1.Id) + .Concat(_eventData.GenerateEvents(80, organization2.Id, project2.Id, stack2.Id)).ToList(); + await _eventRepository.AddAsync(events, o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert - All events preserved (both organizations exist) + var totalAfter = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); + Assert.Equal(160, totalAfter); + } + + [Fact] + public async Task DeleteOrphanedEventsByOrganization_WithOrphanedOrganization_DeletesEventsForMissingOrganization() + { + // Arrange - Valid organization1 + events referencing non-existent organization + var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); + await _organizationRepository.AddAsync(organization1, o => o.ImmediateConsistency()); + + var project = _projectData.GenerateProject(id: TestConstants.ProjectId, organizationId: organization1.Id); + await _projectRepository.AddAsync(project, o => o.ImmediateConsistency()); + + var stack = _stackData.GenerateStack(id: TestConstants.StackId, organizationId: organization1.Id, projectId: project.Id); + await _stackRepository.AddAsync(stack, o => o.ImmediateConsistency()); + + // Valid events + var validEvents = _eventData.GenerateEvents(100, organization1.Id, project.Id, stack.Id).ToList(); + + // Orphaned events referencing a non-existent organization + string fakeOrganizationId = ObjectId.GenerateNewId().ToString(); + string fakeProjectId = ObjectId.GenerateNewId().ToString(); + string fakeStackId = ObjectId.GenerateNewId().ToString(); + var orphanedEvents = _eventData.GenerateEvents(50, fakeOrganizationId, fakeProjectId, fakeStackId).ToList(); + + await _eventRepository.AddAsync(validEvents.Concat(orphanedEvents), o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert - Only valid events survive + var totalAfter = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); + Assert.Equal(100, totalAfter); + } + + [Fact] + public async Task DeleteOrphanedEventsByOrganization_TwoTenantsOneDeleted_OnlyDeletesOrphanedTenantEvents() + { + // Arrange - Organization 1 exists, Organization 2 does NOT exist (never created) but has events + var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); + await _organizationRepository.AddAsync(organization1, o => o.ImmediateConsistency()); + + var project1 = _projectData.GenerateProject(id: TestConstants.ProjectId, organizationId: organization1.Id); + await _projectRepository.AddAsync(project1, o => o.ImmediateConsistency()); + + var stack1 = _stackData.GenerateStack(id: TestConstants.StackId, organizationId: organization1.Id, projectId: project1.Id); + await _stackRepository.AddAsync(stack1, o => o.ImmediateConsistency()); + + // Tenant 1 valid events + var validEvents = _eventData.GenerateEvents(120, organization1.Id, project1.Id, stack1.Id).ToList(); + + // Tenant 2 events (organization doesn't exist, simulates post-hard-delete orphans) + string ghostOrganizationId = ObjectId.GenerateNewId().ToString(); + string ghostProjectId = ObjectId.GenerateNewId().ToString(); + string ghostStackId = ObjectId.GenerateNewId().ToString(); + var ghostEvents = _eventData.GenerateEvents(80, ghostOrganizationId, ghostProjectId, ghostStackId).ToList(); + + await _eventRepository.AddAsync(validEvents.Concat(ghostEvents), o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert + var totalAfter = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); + Assert.Equal(120, totalAfter); + } + + [Fact] + public async Task FixDuplicateStacks_WithDuplicatesAcrossTenants_MergesCorrectly() + { + // Arrange - Two stacks in the same project with the same signature (duplicate) + var organization = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); + await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); + + var project = _projectData.GenerateProject(id: TestConstants.ProjectId, organizationId: organization.Id); + await _projectRepository.AddAsync(project, o => o.ImmediateConsistency()); + + string signatureHash = "abc123def456"; + var stack1 = _stackData.GenerateStack(generateId: true, organizationId: organization.Id, projectId: project.Id, signatureHash: signatureHash); + stack1.CreatedUtc = DateTime.UtcNow.AddDays(-10); + stack1.TotalOccurrences = 5; + var stack2 = _stackData.GenerateStack(generateId: true, organizationId: organization.Id, projectId: project.Id, signatureHash: signatureHash); + stack2.CreatedUtc = DateTime.UtcNow.AddDays(-5); + stack2.TotalOccurrences = 10; + await _stackRepository.AddAsync([stack1, stack2], o => o.ImmediateConsistency()); + + // Events on both stacks + var events1 = _eventData.GenerateEvents(3, organization.Id, project.Id, stack1.Id).ToList(); + var events2 = _eventData.GenerateEvents(7, organization.Id, project.Id, stack2.Id).ToList(); + await _eventRepository.AddAsync(events1.Concat(events2), o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert - One stack should be deleted, all events should point to the surviving stack + await RefreshDataAsync(); + var allStacks = await _stackRepository.GetAllAsync(o => o.IncludeSoftDeletes()); + var activeStacks = allStacks.Documents.Where(s => !s.IsDeleted).ToList(); + var deletedStacks = allStacks.Documents.Where(s => s.IsDeleted).ToList(); + Assert.Single(activeStacks); + Assert.Single(deletedStacks); + + // All events should now reference the surviving stack + var allEvents = await _eventRepository.GetAllAsync(); + Assert.Equal(10, allEvents.Total); + Assert.All(allEvents.Documents, e => Assert.Equal(activeStacks[0].Id, e.StackId)); + } + + [Fact] + public async Task FixDuplicateStacks_NoDuplicates_DoesNotModifyAnything() + { + // Arrange - Two stacks with different signatures across two tenants + var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); + var organization2 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId2); + await _organizationRepository.AddAsync([organization1, organization2], o => o.ImmediateConsistency()); + + var project1 = _projectData.GenerateProject(id: TestConstants.ProjectId, organizationId: organization1.Id); + var project2 = _projectData.GenerateProject(generateId: true, organizationId: organization2.Id); + await _projectRepository.AddAsync([project1, project2], o => o.ImmediateConsistency()); + + var stack1 = _stackData.GenerateStack(generateId: true, organizationId: organization1.Id, projectId: project1.Id, signatureHash: "unique_hash_1"); + var stack2 = _stackData.GenerateStack(generateId: true, organizationId: organization2.Id, projectId: project2.Id, signatureHash: "unique_hash_2"); + await _stackRepository.AddAsync([stack1, stack2], o => o.ImmediateConsistency()); + + var events1 = _eventData.GenerateEvents(20, organization1.Id, project1.Id, stack1.Id).ToList(); + var events2 = _eventData.GenerateEvents(20, organization2.Id, project2.Id, stack2.Id).ToList(); + await _eventRepository.AddAsync(events1.Concat(events2), o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert - Nothing deleted + var allStacks = await _stackRepository.GetAllAsync(o => o.IncludeSoftDeletes()); + Assert.Equal(2, allStacks.Total); + Assert.All(allStacks.Documents, s => Assert.False(s.IsDeleted)); + + var totalEvents = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); + Assert.Equal(40, totalEvents); + } + + [Fact] + public async Task RunAsync_AllOrphanTypes_CleansUpCorrectly() + { + // Arrange - Complex scenario: valid data, orphaned by stack, orphaned by project, orphaned by organization + var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); + await _organizationRepository.AddAsync(organization1, o => o.ImmediateConsistency()); + + var project1 = _projectData.GenerateProject(id: TestConstants.ProjectId, organizationId: organization1.Id); + await _projectRepository.AddAsync(project1, o => o.ImmediateConsistency()); + + var validStack = _stackData.GenerateStack(id: TestConstants.StackId, organizationId: organization1.Id, projectId: project1.Id); + await _stackRepository.AddAsync(validStack, o => o.ImmediateConsistency()); + + // 100 valid events + var validEvents = _eventData.GenerateEvents(100, organization1.Id, project1.Id, validStack.Id).ToList(); + + // 25 orphaned by stack (stack doesn't exist) + string fakeStack = ObjectId.GenerateNewId().ToString(); + var orphanedByStack = _eventData.GenerateEvents(25, organization1.Id, project1.Id, fakeStack).ToList(); + + // 25 orphaned by project (project doesn't exist) + string fakeProject = ObjectId.GenerateNewId().ToString(); + string fakeStack2 = ObjectId.GenerateNewId().ToString(); + var orphanedByProject = _eventData.GenerateEvents(25, organization1.Id, fakeProject, fakeStack2).ToList(); + + // 25 orphaned by organization (organization doesn't exist) + string fakeOrganizationId = ObjectId.GenerateNewId().ToString(); + string fakeProject2 = ObjectId.GenerateNewId().ToString(); + string fakeStack3 = ObjectId.GenerateNewId().ToString(); + var orphanedByOrganization = _eventData.GenerateEvents(25, fakeOrganizationId, fakeProject2, fakeStack3).ToList(); + + await _eventRepository.AddAsync(validEvents.Concat(orphanedByStack).Concat(orphanedByProject).Concat(orphanedByOrganization), o => o.ImmediateConsistency()); + + var totalBefore = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); + Assert.Equal(175, totalBefore); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert - Only 100 valid events remain + var totalAfter = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); + Assert.Equal(100, totalAfter); + } + + [Fact] + public async Task RunAsync_NoOrphans_PreservesEverything() + { + // Arrange - Two complete tenants, no orphans anywhere + var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); + var organization2 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId2); + await _organizationRepository.AddAsync([organization1, organization2], o => o.ImmediateConsistency()); + + var project1 = _projectData.GenerateProject(id: TestConstants.ProjectId, organizationId: organization1.Id); + var project2 = _projectData.GenerateProject(generateId: true, organizationId: organization2.Id); + await _projectRepository.AddAsync([project1, project2], o => o.ImmediateConsistency()); + + var stack1 = _stackData.GenerateStack(id: TestConstants.StackId, organizationId: organization1.Id, projectId: project1.Id); + var stack2 = _stackData.GenerateStack(generateId: true, organizationId: organization2.Id, projectId: project2.Id); + await _stackRepository.AddAsync([stack1, stack2], o => o.ImmediateConsistency()); + + var events1 = _eventData.GenerateEvents(200, organization1.Id, project1.Id, stack1.Id).ToList(); + var events2 = _eventData.GenerateEvents(200, organization2.Id, project2.Id, stack2.Id).ToList(); + await _eventRepository.AddAsync(events1.Concat(events2), o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert - All 400 events preserved + var totalAfter = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); + Assert.Equal(400, totalAfter); + } + + [Fact] + public async Task RunAsync_EmptyDatabase_CompletesWithoutError() + { + // Arrange - nothing + + // Act & Assert - should not throw + await _job.RunAsync(TestCancellationToken); + + var totalAfter = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); + Assert.Equal(0, totalAfter); + } +} diff --git a/tests/Exceptionless.Tests/Mail/MailerTests.cs b/tests/Exceptionless.Tests/Mail/MailerTests.cs index 5f381bb418..582953a94b 100644 --- a/tests/Exceptionless.Tests/Mail/MailerTests.cs +++ b/tests/Exceptionless.Tests/Mail/MailerTests.cs @@ -1,4 +1,3 @@ -using System.Text.Json; using Exceptionless.Core; using Exceptionless.Core.Billing; using Exceptionless.Core.Extensions; @@ -11,6 +10,7 @@ using Exceptionless.Core.Utility; using Exceptionless.Tests.Utility; using Foundatio.Queues; +using Foundatio.Serializer; using Xunit; namespace Exceptionless.Tests.Mail; @@ -40,7 +40,7 @@ public MailerTests(ITestOutputHelper output) : base(output) _plans = GetService(); if (_mailer is NullMailer) - _mailer = new Mailer(GetService>(), GetService(), GetService(), _options, TimeProvider, Log.CreateLogger()); + _mailer = new Mailer(GetService>(), GetService(), GetService(), _options, TimeProvider, Log.CreateLogger()); } [Fact] diff --git a/tests/Exceptionless.Tests/Mapping/WebHookMapperTests.cs b/tests/Exceptionless.Tests/Mapping/WebHookMapperTests.cs index 0deff4747d..308857073f 100644 --- a/tests/Exceptionless.Tests/Mapping/WebHookMapperTests.cs +++ b/tests/Exceptionless.Tests/Mapping/WebHookMapperTests.cs @@ -1,4 +1,3 @@ -using Exceptionless.Core.Models; using Exceptionless.Web.Mapping; using Exceptionless.Web.Models; using Xunit; diff --git a/tests/Exceptionless.Tests/Migrations/FixDuplicateStacksMigrationTests.cs b/tests/Exceptionless.Tests/Migrations/FixDuplicateStacksMigrationTests.cs index 0636cefb97..4479cec683 100644 --- a/tests/Exceptionless.Tests/Migrations/FixDuplicateStacksMigrationTests.cs +++ b/tests/Exceptionless.Tests/Migrations/FixDuplicateStacksMigrationTests.cs @@ -8,7 +8,6 @@ using Foundatio.Repositories.Migrations; using Foundatio.Repositories.Utility; using Foundatio.Utility; -using Nest; using Xunit; namespace Exceptionless.Tests.Migrations; @@ -58,7 +57,7 @@ public async Task WillMergeDuplicatedStacks() await _eventRepository.AddAsync(_eventData.GenerateEvents(count: 100, stackId: originalStack.Id), o => o.ImmediateConsistency()); await _eventRepository.AddAsync(_eventData.GenerateEvents(count: 10, stackId: duplicateStack.Id), o => o.ImmediateConsistency()); - var results = await _stackRepository.FindAsync(q => q.ElasticFilter(Query.Term(s => s.DuplicateSignature, originalStack.DuplicateSignature))); + var results = await _stackRepository.FindAsync(q => q.FieldEquals(s => s.DuplicateSignature, originalStack.DuplicateSignature)); Assert.Equal(2, results.Total); var migration = GetService(); @@ -67,7 +66,7 @@ public async Task WillMergeDuplicatedStacks() await RefreshDataAsync(); - results = await _stackRepository.FindAsync(q => q.ElasticFilter(Query.Term(s => s.DuplicateSignature, originalStack.DuplicateSignature))); + results = await _stackRepository.FindAsync(q => q.FieldEquals(s => s.DuplicateSignature, originalStack.DuplicateSignature)); Assert.Single(results.Documents); var updatedOriginalStack = await _stackRepository.GetByIdAsync(originalStack.Id, o => o.IncludeSoftDeletes()); @@ -113,7 +112,7 @@ public async Task WillMergeToStackWithMostEvents() await _eventRepository.AddAsync(_eventData.GenerateEvents(count: 10, stackId: originalStack.Id), o => o.ImmediateConsistency()); await _eventRepository.AddAsync(_eventData.GenerateEvents(count: 100, stackId: biggerStack.Id), o => o.ImmediateConsistency()); - var results = await _stackRepository.FindAsync(q => q.ElasticFilter(Query.Term(s => s.DuplicateSignature, originalStack.DuplicateSignature))); + var results = await _stackRepository.FindAsync(q => q.FieldEquals(s => s.DuplicateSignature, originalStack.DuplicateSignature)); Assert.Equal(2, results.Total); var migration = GetService(); @@ -122,7 +121,7 @@ public async Task WillMergeToStackWithMostEvents() await RefreshDataAsync(); - results = await _stackRepository.FindAsync(q => q.ElasticFilter(Query.Term(s => s.DuplicateSignature, originalStack.DuplicateSignature))); + results = await _stackRepository.FindAsync(q => q.FieldEquals(s => s.DuplicateSignature, originalStack.DuplicateSignature)); Assert.Single(results.Documents); var updatedOriginalStack = await _stackRepository.GetByIdAsync(originalStack.Id, o => o.IncludeSoftDeletes()); @@ -164,7 +163,7 @@ public async Task WillNotMergeDuplicatedDeletedStacks() await _stackRepository.AddAsync(new[] { originalStack, duplicateStack }, o => o.ImmediateConsistency()); - var results = await _stackRepository.FindAsync(q => q.ElasticFilter(Query.Term(s => s.DuplicateSignature, originalStack.DuplicateSignature))); + var results = await _stackRepository.FindAsync(q => q.FieldEquals(s => s.DuplicateSignature, originalStack.DuplicateSignature)); Assert.Single(results.Documents); var migration = GetService(); @@ -173,7 +172,7 @@ public async Task WillNotMergeDuplicatedDeletedStacks() await RefreshDataAsync(); - results = await _stackRepository.FindAsync(q => q.ElasticFilter(Query.Term(s => s.DuplicateSignature, originalStack.DuplicateSignature))); + results = await _stackRepository.FindAsync(q => q.FieldEquals(s => s.DuplicateSignature, originalStack.DuplicateSignature)); Assert.Single(results.Documents); var updatedOriginalStack = await _stackRepository.GetByIdAsync(originalStack.Id, o => o.IncludeSoftDeletes()); diff --git a/tests/Exceptionless.Tests/Migrations/SetStackDuplicateSignatureMigrationTests.cs b/tests/Exceptionless.Tests/Migrations/SetStackDuplicateSignatureMigrationTests.cs index 7bca481c55..3ab4463e31 100644 --- a/tests/Exceptionless.Tests/Migrations/SetStackDuplicateSignatureMigrationTests.cs +++ b/tests/Exceptionless.Tests/Migrations/SetStackDuplicateSignatureMigrationTests.cs @@ -1,12 +1,10 @@ using Exceptionless.Core.Migrations; -using Exceptionless.Core.Models; using Exceptionless.Core.Repositories; using Exceptionless.Tests.Utility; using Foundatio.Lock; using Foundatio.Repositories; using Foundatio.Repositories.Migrations; using Foundatio.Utility; -using Nest; using Xunit; namespace Exceptionless.Tests.Migrations; @@ -51,7 +49,7 @@ public async Task WillSetStackDuplicateSignature() Assert.NotEmpty(actualStack.SignatureHash); Assert.Equal($"{actualStack.ProjectId}:{actualStack.SignatureHash}", actualStack.DuplicateSignature); - var results = await _repository.FindAsync(q => q.ElasticFilter(Query.Term(s => s.DuplicateSignature, expectedDuplicateSignature))); + var results = await _repository.FindAsync(q => q.FieldEquals(s => s.DuplicateSignature, expectedDuplicateSignature)); Assert.Single(results.Documents); } } diff --git a/tests/Exceptionless.Tests/Pipeline/EventPipelineTests.cs b/tests/Exceptionless.Tests/Pipeline/EventPipelineTests.cs index 65164d881e..c734ab8df8 100644 --- a/tests/Exceptionless.Tests/Pipeline/EventPipelineTests.cs +++ b/tests/Exceptionless.Tests/Pipeline/EventPipelineTests.cs @@ -1,7 +1,6 @@ using System.Diagnostics; using System.Globalization; using System.Text; -using System.Text.Json; using Exceptionless.Core.Billing; using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; @@ -18,6 +17,7 @@ using Exceptionless.Tests.Utility; using Foundatio.Repositories; using Foundatio.Repositories.Extensions; +using Foundatio.Serializer; using Foundatio.Storage; using McSherry.SemanticVersioning; using Xunit; @@ -39,7 +39,7 @@ public sealed class EventPipelineTests : IntegrationTestsBase private readonly IUserRepository _userRepository; private readonly BillingManager _billingManager; private readonly BillingPlans _plans; - private readonly JsonSerializerOptions _jsonOptions; + private readonly ITextSerializer _serializer; public EventPipelineTests(ITestOutputHelper output, AppWebHostFactory factory) : base(output, factory) { @@ -55,7 +55,7 @@ public EventPipelineTests(ITestOutputHelper output, AppWebHostFactory factory) : _pipeline = GetService(); _billingManager = GetService(); _plans = GetService(); - _jsonOptions = GetService(); + _serializer = GetService(); } protected override async Task ResetDataAsync() @@ -224,19 +224,19 @@ public async Task UpdateAutoSessionLastActivityAsync() var results = await _eventRepository.GetAllAsync(o => o.PageLimit(15)); Assert.Equal(9, results.Total); Assert.Equal(2, results.Documents.Where(e => !String.IsNullOrEmpty(e.GetSessionId())).Select(e => e.GetSessionId()).Distinct().Count()); - Assert.Equal(1, results.Documents.Count(e => e.IsSessionEnd() && e.GetUserIdentity(_jsonOptions)?.Identity == "blake@exceptionless.io")); - Assert.Single(results.Documents.Where(e => !String.IsNullOrEmpty(e.GetSessionId()) && e.GetUserIdentity(_jsonOptions)?.Identity == "eric@exceptionless.io").Select(e => e.GetSessionId()).Distinct()); + Assert.Equal(1, results.Documents.Count(e => e.IsSessionEnd() && e.GetUserIdentity(_serializer, _logger)?.Identity == "blake@exceptionless.io")); + Assert.Single(results.Documents.Where(e => !String.IsNullOrEmpty(e.GetSessionId()) && e.GetUserIdentity(_serializer, _logger)?.Identity == "eric@exceptionless.io").Select(e => e.GetSessionId()).Distinct()); Assert.Equal(1, results.Documents.Count(e => String.IsNullOrEmpty(e.GetSessionId()))); Assert.Equal(1, results.Documents.Count(e => e.IsSessionEnd())); var sessionStarts = results.Documents.Where(e => e.IsSessionStart()).ToList(); Assert.Equal(2, sessionStarts.Count); - var firstUserSessionStartEvents = sessionStarts.Single(e => e.GetUserIdentity(_jsonOptions)?.Identity == "blake@exceptionless.io"); + var firstUserSessionStartEvents = sessionStarts.Single(e => e.GetUserIdentity(_serializer, _logger)?.Identity == "blake@exceptionless.io"); Assert.Equal((decimal)(lastEventDate - firstEventDate).TotalSeconds, firstUserSessionStartEvents.Value); Assert.True(firstUserSessionStartEvents.HasSessionEndTime()); - var secondUserSessionStartEvents = sessionStarts.Single(e => e.GetUserIdentity(_jsonOptions)?.Identity == "eric@exceptionless.io"); + var secondUserSessionStartEvents = sessionStarts.Single(e => e.GetUserIdentity(_serializer, _logger)?.Identity == "eric@exceptionless.io"); Assert.Equal((decimal)(lastEventDate - firstEventDate).TotalSeconds, secondUserSessionStartEvents.Value); Assert.False(secondUserSessionStartEvents.HasSessionEndTime()); } @@ -903,10 +903,10 @@ public async Task EnsureIncludePrivateInformationIsRespectedAsync(bool includePr var context = contexts.Single(); Assert.False(context.HasError); - var requestInfo = context.Event.GetRequestInfo(_jsonOptions); - var environmentInfo = context.Event.GetEnvironmentInfo(_jsonOptions); - var userInfo = context.Event.GetUserIdentity(_jsonOptions); - var userDescription = context.Event.GetUserDescription(_jsonOptions); + var requestInfo = context.Event.GetRequestInfo(_serializer, _logger); + var environmentInfo = context.Event.GetEnvironmentInfo(_serializer, _logger); + var userInfo = context.Event.GetUserIdentity(_serializer, _logger); + var userDescription = context.Event.GetUserDescription(_serializer, _logger); Assert.Equal("/test", requestInfo?.Path); Assert.Equal("Windows", environmentInfo?.OSName); @@ -1177,7 +1177,7 @@ public async Task GeneratePerformanceDataAsync() ev.Data.Remove(key); ev.Data.Remove(Event.KnownDataKeys.UserDescription); - var identity = ev.GetUserIdentity(_jsonOptions); + var identity = ev.GetUserIdentity(_serializer, _logger); if (identity?.Identity is not null) { if (!mappedUsers.ContainsKey(identity.Identity)) @@ -1186,7 +1186,7 @@ public async Task GeneratePerformanceDataAsync() ev.SetUserIdentity(mappedUsers[identity.Identity]); } - var request = ev.GetRequestInfo(_jsonOptions); + var request = ev.GetRequestInfo(_serializer, _logger); if (request is not null) { request.Cookies?.Clear(); @@ -1206,7 +1206,7 @@ public async Task GeneratePerformanceDataAsync() } } - InnerError? error = ev.GetError(_jsonOptions); + InnerError? error = ev.GetError(_serializer, _logger); while (error is not null) { error.Message = RandomData.GetSentence(); @@ -1216,13 +1216,13 @@ public async Task GeneratePerformanceDataAsync() error = error.Inner; } - var environment = ev.GetEnvironmentInfo(_jsonOptions); + var environment = ev.GetEnvironmentInfo(_serializer, _logger); environment?.Data?.Clear(); } // inject random session start events. if (currentBatchCount % 10 == 0) - events.Insert(0, events[0].ToSessionStartEvent(_jsonOptions)); + events.Insert(0, events[0].ToSessionStartEvent(_serializer, _logger)); await storage.SaveObjectAsync(Path.Combine(dataDirectory, $"{currentBatchCount++}.json"), events, TestCancellationToken); } @@ -1287,6 +1287,96 @@ private async Task CreateProjectDataAsync(BillingPlan? plan = null) } } + [Fact] + public async Task ErrorPlugin_SetsTargetInfo_AfterPipelineProcessing() + { + // Arrange - Create an error event with multiple stack frames + var ev = GenerateEvent(type: Event.KnownTypes.Error); + ev.Data = new DataDictionary + { + [Event.KnownDataKeys.Error] = new Error + { + Type = "System.InvalidOperationException", + Message = "Test error for target info", + StackTrace = + [ + new Exceptionless.Core.Models.Data.StackFrame + { + DeclaringNamespace = "TestApp.Services", + DeclaringType = "TestService", + Name = "DoWork" + }, + new Exceptionless.Core.Models.Data.StackFrame + { + DeclaringNamespace = "TestApp.Controllers", + DeclaringType = "HomeController", + Name = "Index" + } + ] + } + }; + + // Act - Run through the pipeline + var context = await _pipeline.RunAsync(ev, _organizationData.GenerateSampleOrganization(_billingManager, _plans), _projectData.GenerateSampleProject()); + Assert.False(context.HasError, context.ErrorMessage); + + // Fetch from ES to verify serialized form + await RefreshDataAsync(); + var stored = await _eventRepository.GetByIdAsync(ev.Id); + Assert.NotNull(stored); + + // Assert - @target should have computed strings, not raw Method object + var error = stored.GetError(_serializer, _logger); + Assert.NotNull(error); + + var targetInfo = error.Data?.GetValue(Error.KnownDataKeys.TargetInfo, _serializer); + Assert.NotNull(targetInfo); + Assert.True(targetInfo.TryGetValue("ExceptionType", out var exceptionType), "@target should contain ExceptionType"); + Assert.Equal("System.InvalidOperationException", exceptionType); + Assert.True(targetInfo.TryGetValue("Method", out var method), "@target should contain Method"); + Assert.Contains("TestService.DoWork", method); + + // Assert - is_signature_target should be set on stack frames (FINDING-3b) + Assert.NotNull(error.StackTrace); + Assert.Equal(2, error.StackTrace.Count); + Assert.True(error.StackTrace[0].IsSignatureTarget, "First frame should be signature target"); + Assert.False(error.StackTrace[1].IsSignatureTarget, "Second frame should not be signature target"); + } + + [Fact] + public async Task SimpleErrorPlugin_SetsTargetInfo_AfterPipelineProcessing() + { + // Arrange - Create a simple error event with type and stack trace string + var ev = GenerateEvent(type: Event.KnownTypes.Error); + ev.Data = new DataDictionary + { + [Event.KnownDataKeys.SimpleError] = new SimpleError + { + Type = "System.ArgumentNullException", + Message = "Value cannot be null", + StackTrace = "at TestApp.Services.UserService.GetUser(String userId)" + } + }; + + // Act - Run through the pipeline + var context = await _pipeline.RunAsync(ev, _organizationData.GenerateSampleOrganization(_billingManager, _plans), _projectData.GenerateSampleProject()); + Assert.False(context.HasError, context.ErrorMessage); + + // Fetch from ES to verify serialized form + await RefreshDataAsync(); + var stored = await _eventRepository.GetByIdAsync(ev.Id); + Assert.NotNull(stored); + + // Assert - @target should exist with ExceptionType from SimpleErrorPlugin + var error = stored.GetSimpleError(_serializer, _logger); + Assert.NotNull(error); + + var targetInfo = error.Data?.GetValue(Error.KnownDataKeys.TargetInfo, _serializer); + Assert.NotNull(targetInfo); + Assert.True(targetInfo.TryGetValue("ExceptionType", out var exceptionType), "@target should contain ExceptionType"); + Assert.Equal("System.ArgumentNullException", exceptionType); + } + private PersistentEvent GenerateEvent(DateTimeOffset? occurrenceDate = null, string? userIdentity = null, string? type = null, string? sessionId = null) { occurrenceDate ??= DateTimeOffset.Now; diff --git a/tests/Exceptionless.Tests/Plugins/EventParserTests.cs b/tests/Exceptionless.Tests/Plugins/EventParserTests.cs index 5e0f215f23..544aa92828 100644 --- a/tests/Exceptionless.Tests/Plugins/EventParserTests.cs +++ b/tests/Exceptionless.Tests/Plugins/EventParserTests.cs @@ -1,7 +1,8 @@ -using Exceptionless.Core.Extensions; +using System.Text.Json; using Exceptionless.Core.Models; using Exceptionless.Core.Plugins.EventParser; -using Newtonsoft.Json; +using Exceptionless.Tests.Utility; +using Foundatio.Serializer; using Xunit; namespace Exceptionless.Tests.Plugins; @@ -9,10 +10,14 @@ namespace Exceptionless.Tests.Plugins; public sealed class EventParserTests : TestWithServices { private readonly EventParserPluginManager _parser; + private readonly ITextSerializer _serializer; + private readonly JsonSerializerOptions _jsonOptions; public EventParserTests(ITestOutputHelper output) : base(output) { _parser = GetService(); + _serializer = GetService(); + _jsonOptions = GetService(); } public static IEnumerable EventData => new[] { @@ -52,9 +57,14 @@ public void VerifyEventParserSerialization(string eventsFilePath) var events = _parser.ParseEvents(json, 2, "exceptionless/2.0.0.0"); Assert.Single(events); + var ev = events.Single(); + // Verify structural equivalence: parse → serialize should produce + // content equivalent to the original file (ignoring nulls and empty collections + // that STJ's WhenWritingNull and EmptyCollectionModifier skip). string expectedContent = File.ReadAllText(eventsFilePath); - Assert.Equal(expectedContent, events.First().ToJson(Formatting.Indented, GetService())); + string actualContent = JsonSerializer.Serialize(ev, _jsonOptions); + JsonAssert.AssertJsonEquivalent(expectedContent, actualContent); } [Theory] @@ -63,7 +73,7 @@ public void CanDeserializeEvents(string eventsFilePath) { string json = File.ReadAllText(eventsFilePath); - var ev = json.FromJson(GetService()); + var ev = _serializer.Deserialize(json); Assert.NotNull(ev); } diff --git a/tests/Exceptionless.Tests/Plugins/EventUpgraderTests.cs b/tests/Exceptionless.Tests/Plugins/EventUpgraderTests.cs index 2459e1b4ac..416fcce9e2 100644 --- a/tests/Exceptionless.Tests/Plugins/EventUpgraderTests.cs +++ b/tests/Exceptionless.Tests/Plugins/EventUpgraderTests.cs @@ -1,5 +1,8 @@ +using System.Text.Json; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Plugins.EventParser; using Exceptionless.Core.Plugins.EventUpgrader; +using Exceptionless.Tests.Utility; using Xunit; namespace Exceptionless.Tests.Plugins; @@ -8,11 +11,13 @@ public sealed class EventUpgraderTests : TestWithServices { private readonly EventUpgraderPluginManager _upgrader; private readonly EventParserPluginManager _parser; + private readonly JsonSerializerOptions _jsonOptions; public EventUpgraderTests(ITestOutputHelper output) : base(output) { _upgrader = GetService(); _parser = GetService(); + _jsonOptions = GetService(); } [Theory] @@ -24,9 +29,9 @@ public void ParseErrors(string errorFilePath) _upgrader.Upgrade(ctx); string expectedContent = File.ReadAllText(Path.ChangeExtension(errorFilePath, ".expected.json")); - Assert.Equal(expectedContent, ctx.Documents.First?.ToString()); + JsonAssert.AssertJsonEquivalent(expectedContent, ctx.Documents.First().ToFormattedString(_jsonOptions)); - var events = _parser.ParseEvents(ctx.Documents.ToString(), 2, "exceptionless/2.0.0.0"); + var events = _parser.ParseEvents(ctx.Documents.ToFormattedString(_jsonOptions), 2, "exceptionless/2.0.0.0"); Assert.Single(events); } diff --git a/tests/Exceptionless.Tests/Plugins/GeoTests.cs b/tests/Exceptionless.Tests/Plugins/GeoTests.cs index fe6c576fb5..70d9b06953 100644 --- a/tests/Exceptionless.Tests/Plugins/GeoTests.cs +++ b/tests/Exceptionless.Tests/Plugins/GeoTests.cs @@ -1,5 +1,4 @@ using System.Diagnostics; -using System.Text.Json; using Exceptionless.Core; using Exceptionless.Core.Billing; using Exceptionless.Core.Geo; @@ -13,6 +12,7 @@ using Exceptionless.Tests.Utility; using Foundatio.Caching; using Foundatio.Resilience; +using Foundatio.Serializer; using Foundatio.Storage; using Xunit; @@ -29,7 +29,7 @@ public sealed class GeoTests : TestWithServices private readonly AppOptions _options; private readonly OrganizationData _organizationData; private readonly ProjectData _projectData; - private readonly JsonSerializerOptions _jsonOptions; + private readonly ITextSerializer _serializer; public GeoTests(ITestOutputHelper output) : base(output) { @@ -38,7 +38,7 @@ public GeoTests(ITestOutputHelper output) : base(output) _options = GetService(); _organizationData = GetService(); _projectData = GetService(); - _jsonOptions = GetService(); + _serializer = GetService(); } private async Task GetResolverAsync(ILoggerFactory loggerFactory) @@ -74,12 +74,12 @@ public async Task WillNotSetLocation() if (resolver is NullGeoIpService) return; - var plugin = new GeoPlugin(resolver, _jsonOptions, _options, Log); + var plugin = new GeoPlugin(resolver, _serializer, _options, Log); var ev = new PersistentEvent { Geo = GREEN_BAY_COORDINATES }; await plugin.EventBatchProcessingAsync(new List { new(ev, _organizationData.GenerateSampleOrganization(_billingManager, _plans), _projectData.GenerateSampleProject()) }); Assert.Equal(GREEN_BAY_COORDINATES, ev.Geo); - Assert.Null(ev.GetLocation(_jsonOptions)); + Assert.Null(ev.GetLocation(_serializer, _logger)); } [Theory] @@ -94,12 +94,12 @@ public async Task WillResetLocation(string? geo) if (resolver is NullGeoIpService) return; - var plugin = new GeoPlugin(resolver, _jsonOptions, _options, Log); + var plugin = new GeoPlugin(resolver, _serializer, _options, Log); var ev = new PersistentEvent { Geo = geo }; await plugin.EventBatchProcessingAsync(new List { new(ev, _organizationData.GenerateSampleOrganization(_billingManager, _plans), _projectData.GenerateSampleProject()) }); Assert.Null(ev.Geo); - Assert.Null(ev.GetLocation(_jsonOptions)); + Assert.Null(ev.GetLocation(_serializer, _logger)); } [Fact] @@ -109,14 +109,14 @@ public async Task WillSetLocationFromGeo() if (resolver is NullGeoIpService) return; - var plugin = new GeoPlugin(resolver, _jsonOptions, _options, Log); + var plugin = new GeoPlugin(resolver, _serializer, _options, Log); var ev = new PersistentEvent { Geo = GREEN_BAY_IP }; await plugin.EventBatchProcessingAsync(new List { new(ev, _organizationData.GenerateSampleOrganization(_billingManager, _plans), _projectData.GenerateSampleProject()) }); Assert.NotNull(ev.Geo); Assert.NotEqual(GREEN_BAY_IP, ev.Geo); - var location = ev.GetLocation(_jsonOptions); + var location = ev.GetLocation(_serializer, _logger); Assert.Equal("US", location?.Country); Assert.Equal("WI", location?.Level1); Assert.Equal("Green Bay", location?.Locality); @@ -129,14 +129,14 @@ public async Task WillSetLocationFromRequestInfo() if (resolver is NullGeoIpService) return; - var plugin = new GeoPlugin(resolver, _jsonOptions, _options, Log); + var plugin = new GeoPlugin(resolver, _serializer, _options, Log); var ev = new PersistentEvent(); ev.AddRequestInfo(new RequestInfo { ClientIpAddress = GREEN_BAY_IP }); await plugin.EventBatchProcessingAsync(new List { new(ev, _organizationData.GenerateSampleOrganization(_billingManager, _plans), _projectData.GenerateSampleProject()) }); Assert.NotNull(ev.Geo); - var location = ev.GetLocation(_jsonOptions); + var location = ev.GetLocation(_serializer, _logger); Assert.Equal("US", location?.Country); Assert.Equal("WI", location?.Level1); Assert.Equal("Green Bay", location?.Locality); @@ -149,14 +149,14 @@ public async Task WillSetLocationFromEnvironmentInfoInfo() if (resolver is NullGeoIpService) return; - var plugin = new GeoPlugin(resolver, _jsonOptions, _options, Log); + var plugin = new GeoPlugin(resolver, _serializer, _options, Log); var ev = new PersistentEvent(); ev.SetEnvironmentInfo(new EnvironmentInfo { IpAddress = $"127.0.0.1,{GREEN_BAY_IP}" }); await plugin.EventBatchProcessingAsync(new List { new(ev, _organizationData.GenerateSampleOrganization(_billingManager, _plans), _projectData.GenerateSampleProject()) }); Assert.NotNull(ev.Geo); - var location = ev.GetLocation(_jsonOptions); + var location = ev.GetLocation(_serializer, _logger); Assert.Equal("US", location?.Country); Assert.Equal("WI", location?.Level1); Assert.Equal("Green Bay", location?.Locality); @@ -169,7 +169,7 @@ public async Task WillSetFromSingleGeo() if (resolver is NullGeoIpService) return; - var plugin = new GeoPlugin(resolver, _jsonOptions, _options, Log); + var plugin = new GeoPlugin(resolver, _serializer, _options, Log); var contexts = new List { new(new PersistentEvent { Geo = GREEN_BAY_IP }, _organizationData.GenerateSampleOrganization(_billingManager, _plans), _projectData.GenerateSampleProject()), @@ -182,7 +182,7 @@ public async Task WillSetFromSingleGeo() { AssertCoordinatesAreEqual(GREEN_BAY_COORDINATES, context.Event.Geo); - var location = context.Event.GetLocation(_jsonOptions); + var location = context.Event.GetLocation(_serializer, _logger); Assert.Equal("US", location?.Country); Assert.Equal("WI", location?.Level1); Assert.Equal("Green Bay", location?.Locality); @@ -196,7 +196,7 @@ public async Task WillNotSetFromMultipleGeo() if (resolver is NullGeoIpService) return; - var plugin = new GeoPlugin(resolver, _jsonOptions, _options, Log); + var plugin = new GeoPlugin(resolver, _serializer, _options, Log); var ev = new PersistentEvent(); var greenBayEvent = new PersistentEvent { Geo = GREEN_BAY_IP }; @@ -208,13 +208,13 @@ await plugin.EventBatchProcessingAsync(new List { }); AssertCoordinatesAreEqual(GREEN_BAY_COORDINATES, greenBayEvent.Geo); - var location = greenBayEvent.GetLocation(_jsonOptions); + var location = greenBayEvent.GetLocation(_serializer, _logger); Assert.Equal("US", location?.Country); Assert.Equal("WI", location?.Level1); Assert.Equal("Green Bay", location?.Locality); AssertCoordinatesAreEqual(IRVING_COORDINATES, irvingEvent.Geo); - location = irvingEvent.GetLocation(_jsonOptions); + location = irvingEvent.GetLocation(_serializer, _logger); Assert.Equal("US", location?.Country); Assert.Equal("TX", location?.Level1); Assert.Equal("Irving", location?.Locality); @@ -242,7 +242,7 @@ public async Task WillSetMultipleFromEmptyGeo() if (resolver is NullGeoIpService) return; - var plugin = new GeoPlugin(resolver, _jsonOptions, _options, Log); + var plugin = new GeoPlugin(resolver, _serializer, _options, Log); var ev = new PersistentEvent(); var greenBayEvent = new PersistentEvent(); @@ -256,13 +256,13 @@ await plugin.EventBatchProcessingAsync(new List { }); AssertCoordinatesAreEqual(GREEN_BAY_COORDINATES, greenBayEvent.Geo); - var location = greenBayEvent.GetLocation(_jsonOptions); + var location = greenBayEvent.GetLocation(_serializer, _logger); Assert.Equal("US", location?.Country); Assert.Equal("WI", location?.Level1); Assert.Equal("Green Bay", location?.Locality); AssertCoordinatesAreEqual(IRVING_COORDINATES, irvingEvent.Geo); - location = irvingEvent.GetLocation(_jsonOptions); + location = irvingEvent.GetLocation(_serializer, _logger); Assert.Equal("US", location?.Country); Assert.Equal("TX", location?.Level1); Assert.Equal("Irving", location?.Locality); diff --git a/tests/Exceptionless.Tests/Plugins/SummaryDataTests.cs b/tests/Exceptionless.Tests/Plugins/SummaryDataTests.cs index 9e0ffad6e6..e675258566 100644 --- a/tests/Exceptionless.Tests/Plugins/SummaryDataTests.cs +++ b/tests/Exceptionless.Tests/Plugins/SummaryDataTests.cs @@ -1,26 +1,31 @@ -using Exceptionless.Core.Extensions; +using System.Text.Json; using Exceptionless.Core.Models; using Exceptionless.Core.Plugins.Formatting; -using Newtonsoft.Json; +using Exceptionless.Tests.Utility; +using Foundatio.Serializer; using Xunit; namespace Exceptionless.Tests.Plugins; public class SummaryDataTests : TestWithServices { - public SummaryDataTests(ITestOutputHelper output) : base(output) { } + private readonly ITextSerializer _serializer; + private readonly JsonSerializerOptions _jsonOptions; + + public SummaryDataTests(ITestOutputHelper output) : base(output) + { + _serializer = GetService(); + _jsonOptions = GetService(); + } [Theory] [MemberData(nameof(Events))] public async Task EventSummaryData(string path) { - var settings = GetService(); - settings.Formatting = Formatting.Indented; - string json = await File.ReadAllTextAsync(path, TestCancellationToken); Assert.NotNull(json); - var ev = json.FromJson(settings); + var ev = _serializer.Deserialize(json); Assert.NotNull(ev); var data = GetService().GetEventSummaryData(ev); @@ -33,20 +38,17 @@ public async Task EventSummaryData(string path) }; string expectedContent = await File.ReadAllTextAsync(Path.ChangeExtension(path, "summary.json"), TestCancellationToken); - Assert.Equal(expectedContent, JsonConvert.SerializeObject(summary, settings)); + JsonAssert.AssertJsonEquivalent(expectedContent, JsonSerializer.Serialize(summary, _jsonOptions)); } [Theory] [MemberData(nameof(Stacks))] public async Task StackSummaryData(string path) { - var settings = GetService(); - settings.Formatting = Formatting.Indented; - string json = await File.ReadAllTextAsync(path, TestCancellationToken); Assert.NotNull(json); - var stack = json.FromJson(settings); + var stack = _serializer.Deserialize(json); Assert.NotNull(stack); var data = GetService().GetStackSummaryData(stack); @@ -61,7 +63,7 @@ public async Task StackSummaryData(string path) }; string expectedContent = await File.ReadAllTextAsync(Path.ChangeExtension(path, "summary.json"), TestCancellationToken); - Assert.Equal(expectedContent, JsonConvert.SerializeObject(summary, settings)); + JsonAssert.AssertJsonEquivalent(expectedContent, JsonSerializer.Serialize(summary, _jsonOptions)); } public static IEnumerable Events diff --git a/tests/Exceptionless.Tests/Plugins/WebHookDataTests.cs b/tests/Exceptionless.Tests/Plugins/WebHookDataTests.cs index 86e353f1e0..f0872829ae 100644 --- a/tests/Exceptionless.Tests/Plugins/WebHookDataTests.cs +++ b/tests/Exceptionless.Tests/Plugins/WebHookDataTests.cs @@ -1,15 +1,19 @@ +using System.Text.Json; using Exceptionless.Core.Billing; using Exceptionless.Core.Models; using Exceptionless.Core.Plugins.Formatting; using Exceptionless.Core.Plugins.WebHook; using Exceptionless.Tests.Utility; -using Newtonsoft.Json; +using Foundatio.Serializer; using Xunit; namespace Exceptionless.Tests.Plugins; public sealed class WebHookDataTests : TestWithServices { + private readonly ITextSerializer _serializer; + private readonly JsonSerializerOptions _jsonOptions; + private readonly JsonSerializerOptions _versionOneJsonOptions; private readonly OrganizationData _organizationData; private readonly ProjectData _projectData; private readonly StackData _stackData; @@ -18,6 +22,9 @@ public sealed class WebHookDataTests : TestWithServices public WebHookDataTests(ITestOutputHelper output) : base(output) { + _serializer = GetService(); + _jsonOptions = GetService(); + _versionOneJsonOptions = VersionOnePlugin.CreateJsonSerializerOptions(_jsonOptions); _organizationData = GetService(); _projectData = GetService(); _stackData = GetService(); @@ -29,15 +36,13 @@ public WebHookDataTests(ITestOutputHelper output) : base(output) [MemberData(nameof(WebHookData))] public async Task CreateFromEventAsync(string version, bool expectData) { - var settings = GetService(); - settings.Formatting = Formatting.Indented; object? data = await _webHookData.CreateFromEventAsync(GetWebHookDataContext(version)); if (expectData) { + Assert.NotNull(data); string filePath = Path.GetFullPath(Path.Combine("..", "..", "..", "Plugins", "WebHookData", $"{version}.event.expected.json")); string expectedContent = await File.ReadAllTextAsync(filePath, TestCancellationToken); - string actualContent = JsonConvert.SerializeObject(data, settings); - Assert.Equal(expectedContent, actualContent); + JsonAssert.AssertJsonEquals(expectedContent, JsonSerializer.Serialize(data, GetJsonSerializerOptions(data))); } else { @@ -49,15 +54,13 @@ public async Task CreateFromEventAsync(string version, bool expectData) [MemberData(nameof(WebHookData))] public async Task CanCreateFromStackAsync(string version, bool expectData) { - var settings = GetService(); - settings.Formatting = Formatting.Indented; object? data = await _webHookData.CreateFromStackAsync(GetWebHookDataContext(version)); if (expectData) { + Assert.NotNull(data); string filePath = Path.GetFullPath(Path.Combine("..", "..", "..", "Plugins", "WebHookData", $"{version}.stack.expected.json")); string expectedContent = await File.ReadAllTextAsync(filePath, TestCancellationToken); - string actualContent = JsonConvert.SerializeObject(data, settings); - Assert.Equal(expectedContent, actualContent); + JsonAssert.AssertJsonEquals(expectedContent, JsonSerializer.Serialize(data, GetJsonSerializerOptions(data))); } else { @@ -72,13 +75,17 @@ public async Task CanCreateFromStackAsync(string version, bool expectData) new object[] { "v3", false } }.ToArray(); + private JsonSerializerOptions GetJsonSerializerOptions(object data) + { + return data is VersionOnePlugin.VersionOneWebHookEvent or VersionOnePlugin.VersionOneWebHookStack + ? _versionOneJsonOptions + : _jsonOptions; + } + private WebHookDataContext GetWebHookDataContext(string version) { string json = File.ReadAllText(Path.GetFullPath(Path.Combine("..", "..", "..", "ErrorData", "1477.expected.json"))); - var settings = GetService(); - settings.Formatting = Formatting.Indented; - var hook = new WebHook { Id = TestConstants.WebHookId, @@ -93,7 +100,7 @@ private WebHookDataContext GetWebHookDataContext(string version) var organization = _organizationData.GenerateSampleOrganization(GetService(), GetService()); var project = _projectData.GenerateSampleProject(); - var ev = JsonConvert.DeserializeObject(json, settings); + var ev = _serializer.Deserialize(json); Assert.NotNull(ev); ev.OrganizationId = TestConstants.OrganizationId; ev.ProjectId = TestConstants.ProjectId; diff --git a/tests/Exceptionless.Tests/Repositories/Configuration/EventIndexFieldNameTests.cs b/tests/Exceptionless.Tests/Repositories/Configuration/EventIndexFieldNameTests.cs new file mode 100644 index 0000000000..a910de4f78 --- /dev/null +++ b/tests/Exceptionless.Tests/Repositories/Configuration/EventIndexFieldNameTests.cs @@ -0,0 +1,36 @@ +using Exceptionless.Core.Models; +using Exceptionless.Core.Models.Data; +using Exceptionless.Core.Repositories.Configuration; +using Foundatio.Xunit; +using Xunit; + +namespace Exceptionless.Tests.Repositories.Configuration; + +public class EventIndexFieldNameTests : TestWithLoggingBase +{ + public EventIndexFieldNameTests(ITestOutputHelper output) : base(output) { } + + [Fact] + public void DataPath_EnvironmentInfoOSName_UsesJsonPropertyNameOverride() + { + string path = EventIndexExtensions.DataPath(Event.KnownDataKeys.EnvironmentInfo, e => e.OSName); + + Assert.Equal("data.@environment.o_s_name", path); + } + + [Fact] + public void DataPath_RequestInfoClientIpAddress_UsesSerializerNamingPolicy() + { + string path = EventIndexExtensions.DataPath(Event.KnownDataKeys.RequestInfo, r => r.ClientIpAddress); + + Assert.Equal("data.@request.client_ip_address", path); + } + + [Fact] + public void DataDictionaryPath_RequestInfoData_AppendsKnownDictionaryKey() + { + string path = EventIndexExtensions.DataDictionaryPath(Event.KnownDataKeys.RequestInfo, r => r.Data, RequestInfo.KnownDataKeys.BrowserVersion); + + Assert.Equal("data.@request.data.@browser_version", path); + } +} diff --git a/tests/Exceptionless.Tests/Repositories/EventRepositoryTests.cs b/tests/Exceptionless.Tests/Repositories/EventRepositoryTests.cs index cd790c66b3..95c5d2b628 100644 --- a/tests/Exceptionless.Tests/Repositories/EventRepositoryTests.cs +++ b/tests/Exceptionless.Tests/Repositories/EventRepositoryTests.cs @@ -1,5 +1,4 @@ using System.Diagnostics; -using System.Text.Json; using Exceptionless.Core.Models; using Exceptionless.Core.Models.Data; using Exceptionless.Core.Repositories; @@ -8,6 +7,7 @@ using Exceptionless.Tests.Utility; using Foundatio.Repositories; using Foundatio.Repositories.Utility; +using Foundatio.Serializer; using Xunit; using LogLevel = Microsoft.Extensions.Logging.LogLevel; @@ -21,7 +21,7 @@ public sealed class EventRepositoryTests : IntegrationTestsBase private readonly IEventRepository _repository; private readonly StackData _stackData; private readonly IStackRepository _stackRepository; - private readonly JsonSerializerOptions _jsonOptions; + private readonly ITextSerializer _serializer; public EventRepositoryTests(ITestOutputHelper output, AppWebHostFactory factory) : base(output, factory) { @@ -30,10 +30,10 @@ public EventRepositoryTests(ITestOutputHelper output, AppWebHostFactory factory) _repository = GetService(); _stackData = GetService(); _stackRepository = GetService(); - _jsonOptions = GetService(); + _serializer = GetService(); } - [Fact(Skip = "https://github.com/elastic/elasticsearch-net/issues/2463")] + [Fact] public async Task GetAsync() { Log.SetLogLevel(LogLevel.Trace); @@ -50,7 +50,17 @@ public async Task GetAsync() Geo = "40,-70" }); - Assert.Equal(ev, await _repository.GetByIdAsync(ev.Id)); + var actual = await _repository.GetByIdAsync(ev.Id); + Assert.NotNull(actual); + Assert.Equal(ev.Id, actual.Id); + Assert.Equal(ev.Type, actual.Type); + Assert.Equal(ev.OrganizationId, actual.OrganizationId); + Assert.Equal(ev.ProjectId, actual.ProjectId); + Assert.Equal(ev.StackId, actual.StackId); + Assert.Equal(ev.Date, actual.Date); + Assert.Equal(ev.Count, actual.Count); + Assert.Equal(ev.Value, actual.Value); + Assert.Equal(ev.Geo, actual.Geo); } [Fact(Skip = "Performance Testing")] @@ -224,7 +234,7 @@ public async Task RemoveAllByClientIpAndDateAsync() Assert.Equal(NUMBER_OF_EVENTS_TO_CREATE, events.Count); events.ForEach(e => { - var ri = e.GetRequestInfo(_jsonOptions); + var ri = e.GetRequestInfo(_serializer, _logger); Assert.NotNull(ri); Assert.Equal(_clientIpAddress, ri.ClientIpAddress); }); diff --git a/tests/Exceptionless.Tests/Repositories/OrganizationRepositoryTests.cs b/tests/Exceptionless.Tests/Repositories/OrganizationRepositoryTests.cs index 94a4144e79..3e1097279f 100644 --- a/tests/Exceptionless.Tests/Repositories/OrganizationRepositoryTests.cs +++ b/tests/Exceptionless.Tests/Repositories/OrganizationRepositoryTests.cs @@ -1,6 +1,8 @@ using Exceptionless.Core.Billing; using Exceptionless.Core.Models; using Exceptionless.Core.Repositories; +using Exceptionless.Core.Repositories.Queries; +using Exceptionless.Tests.Utility; using Foundatio.Caching; using Foundatio.Repositories; using Xunit; @@ -61,4 +63,139 @@ public async Task CanAddAndGetByCachedAsync() await _repository.RemoveAllAsync(o => o.ImmediateConsistency()); Assert.Equal(0, _cache.Count); } + + [Fact] + public async Task GetByCriteria_SearchById_ReturnsMatchingOrganization() + { + // Arrange + var organization = new Organization { Name = "Criteria Test Organization", PlanId = _plans.FreePlan.Id }; + await _repository.AddAsync(organization, o => o.ImmediateConsistency()); + + // Act + var results = await _repository.GetByCriteriaAsync(organization.Id, + o => o.PageLimit(10), OrganizationSortBy.Newest); + + // Assert + Assert.Single(results.Documents); + Assert.Equal(organization.Id, results.Documents.First().Id); + } + + [Fact] + public async Task GetByCriteria_SearchByName_ReturnsMatchingOrganization() + { + // Arrange + var organization = new Organization { Name = "Unique Search Name", PlanId = _plans.FreePlan.Id }; + await _repository.AddAsync(organization, o => o.ImmediateConsistency()); + + // Act + var results = await _repository.GetByCriteriaAsync("Unique Search Name", + o => o.PageLimit(10), OrganizationSortBy.Newest); + + // Assert + Assert.Single(results.Documents); + Assert.Equal("Unique Search Name", results.Documents.First().Name); + } + + [Fact] + public async Task GetByFilter_AppFilter_ReturnsOnlyAllowedOrganizations() + { + // Arrange + var organization1 = new Organization { Name = "Allowed Organization", PlanId = _plans.FreePlan.Id }; + var organization2 = new Organization { Name = "Blocked Organization", PlanId = _plans.FreePlan.Id }; + await _repository.AddAsync([organization1, organization2], o => o.ImmediateConsistency()); + + // Act + var results = await _repository.GetByFilterAsync(new AppFilter(organization1), null, null); + + // Assert + Assert.Single(results.Documents); + Assert.Equal(organization1.Id, results.Documents.First().Id); + } + + [Fact] + public async Task GetByCriteria_PaidFilter_ExcludesFreeOrganizations() + { + // Arrange + var freeOrganization = new Organization { Name = "Free Organization", PlanId = _plans.FreePlan.Id }; + var paidOrganization = new Organization { Name = "Paid Organization", PlanId = _plans.SmallPlan.Id }; + await _repository.AddAsync([freeOrganization, paidOrganization], o => o.ImmediateConsistency()); + + // Act + var paidResults = await _repository.GetByCriteriaAsync(null, + o => o.PageLimit(10), OrganizationSortBy.Newest, paid: true); + var freeResults = await _repository.GetByCriteriaAsync(null, + o => o.PageLimit(10), OrganizationSortBy.Newest, paid: false); + + // Assert + Assert.All(paidResults.Documents, d => Assert.NotEqual(_plans.FreePlan.Id, d.PlanId)); + Assert.All(freeResults.Documents, d => Assert.Equal(_plans.FreePlan.Id, d.PlanId)); + } + + [Fact] + public async Task GetByCriteria_SuspendedFilter_ReturnsExpectedOrganizations() + { + // Arrange + var activeOrganization = new Organization { Name = "Active Organization", PlanId = _plans.FreePlan.Id, BillingStatus = BillingStatus.Active }; + var trialingOrganization = new Organization { Name = "Trialing Organization", PlanId = _plans.FreePlan.Id, BillingStatus = BillingStatus.Trialing }; + var canceledOrganization = new Organization { Name = "Canceled Organization", PlanId = _plans.FreePlan.Id, BillingStatus = BillingStatus.Canceled }; + var pastDueOrganization = new Organization { Name = "Past Due Organization", PlanId = _plans.FreePlan.Id, BillingStatus = BillingStatus.PastDue }; + var unpaidOrganization = new Organization { Name = "Unpaid Organization", PlanId = _plans.FreePlan.Id, BillingStatus = BillingStatus.Unpaid }; + var manuallySuspendedOrganization = new Organization + { + Name = "Manual Suspension Organization", + PlanId = _plans.FreePlan.Id, + BillingStatus = BillingStatus.Active, + IsSuspended = true, + SuspensionCode = SuspensionCode.Abuse, + SuspensionDate = DateTime.UtcNow, + SuspendedByUserId = TestConstants.UserId + }; + await _repository.AddAsync([ + activeOrganization, + trialingOrganization, + canceledOrganization, + pastDueOrganization, + unpaidOrganization, + manuallySuspendedOrganization + ], o => o.ImmediateConsistency()); + + // Act + var suspendedResults = await _repository.GetByCriteriaAsync(null, + o => o.PageLimit(10), OrganizationSortBy.Newest, suspended: true); + var activeResults = await _repository.GetByCriteriaAsync(null, + o => o.PageLimit(10), OrganizationSortBy.Newest, suspended: false); + + // Assert + Assert.Contains(suspendedResults.Documents, o => String.Equals(o.Id, pastDueOrganization.Id, StringComparison.Ordinal)); + Assert.Contains(suspendedResults.Documents, o => String.Equals(o.Id, unpaidOrganization.Id, StringComparison.Ordinal)); + Assert.Contains(suspendedResults.Documents, o => String.Equals(o.Id, manuallySuspendedOrganization.Id, StringComparison.Ordinal)); + Assert.DoesNotContain(suspendedResults.Documents, o => String.Equals(o.Id, activeOrganization.Id, StringComparison.Ordinal)); + Assert.DoesNotContain(suspendedResults.Documents, o => String.Equals(o.Id, trialingOrganization.Id, StringComparison.Ordinal)); + Assert.DoesNotContain(suspendedResults.Documents, o => String.Equals(o.Id, canceledOrganization.Id, StringComparison.Ordinal)); + + Assert.Contains(activeResults.Documents, o => String.Equals(o.Id, activeOrganization.Id, StringComparison.Ordinal)); + Assert.Contains(activeResults.Documents, o => String.Equals(o.Id, trialingOrganization.Id, StringComparison.Ordinal)); + Assert.Contains(activeResults.Documents, o => String.Equals(o.Id, canceledOrganization.Id, StringComparison.Ordinal)); + Assert.DoesNotContain(activeResults.Documents, o => String.Equals(o.Id, pastDueOrganization.Id, StringComparison.Ordinal)); + Assert.DoesNotContain(activeResults.Documents, o => String.Equals(o.Id, unpaidOrganization.Id, StringComparison.Ordinal)); + Assert.DoesNotContain(activeResults.Documents, o => String.Equals(o.Id, manuallySuspendedOrganization.Id, StringComparison.Ordinal)); + } + + [Fact] + public async Task GetByCriteria_SortByName_ReturnsSortedResults() + { + // Arrange + var organizationC = new Organization { Name = "Charlie Organization", PlanId = _plans.FreePlan.Id }; + var organizationA = new Organization { Name = "Alpha Organization", PlanId = _plans.FreePlan.Id }; + var organizationB = new Organization { Name = "Bravo Organization", PlanId = _plans.FreePlan.Id }; + await _repository.AddAsync([organizationC, organizationA, organizationB], o => o.ImmediateConsistency()); + + // Act + var results = await _repository.GetByCriteriaAsync(null, + o => o.PageLimit(10), OrganizationSortBy.Alphabetical); + + // Assert + var names = results.Documents.Select(d => d.Name).ToList(); + Assert.Equal(names.OrderBy(n => n), names); + } } diff --git a/tests/Exceptionless.Tests/Repositories/ProjectRepositoryTests.cs b/tests/Exceptionless.Tests/Repositories/ProjectRepositoryTests.cs index 5cf55c0fbf..3db6831f9a 100644 --- a/tests/Exceptionless.Tests/Repositories/ProjectRepositoryTests.cs +++ b/tests/Exceptionless.Tests/Repositories/ProjectRepositoryTests.cs @@ -7,6 +7,7 @@ using Foundatio.Caching; using Foundatio.Repositories; using Foundatio.Repositories.Models; +using Foundatio.Serializer; using Xunit; namespace Exceptionless.Tests.Repositories; @@ -17,6 +18,7 @@ public sealed class ProjectRepositoryTests : IntegrationTestsBase private readonly OrganizationData _organizationData; private readonly ProjectData _projectData; private readonly IProjectRepository _repository; + private readonly ITextSerializer _serializer; public ProjectRepositoryTests(ITestOutputHelper output, AppWebHostFactory factory) : base(output, factory) { @@ -24,6 +26,7 @@ public ProjectRepositoryTests(ITestOutputHelper output, AppWebHostFactory factor _projectData = GetService(); _cache = GetService(); _repository = GetService(); + _serializer = GetService(); } [Fact] @@ -138,7 +141,8 @@ public async Task CanRoundTripWithCaching() var actual = await _repository.GetByIdAsync(project.Id, o => o.Cache()); Assert.NotNull(actual); Assert.Equal(project.Name, actual.Name); - var actualToken = actual.GetSlackToken(); + + var actualToken = actual.GetSlackToken(_serializer); Assert.Equal(token.AccessToken, actualToken?.AccessToken); var actualCache = await _cache.GetAsync>>("Project:" + project.Id); @@ -148,7 +152,29 @@ public async Task CanRoundTripWithCaching() var cachedDoc = cachedDocs.Single(); Assert.NotNull(cachedDoc.Document); Assert.Equal(project.Name, cachedDoc.Document.Name); - var actualCacheToken = actual.GetSlackToken(); + var actualCacheToken = actual.GetSlackToken(_serializer); Assert.Equal(token.AccessToken, actualCacheToken?.AccessToken); } + + [Fact] + public async Task GetByNextSummaryNotificationOffset_FilterExpression_FiltersCorrectly() + { + // Arrange + var pastProject = _projectData.GenerateProject(generateId: true, + organizationId: TestConstants.OrganizationId, name: "Past Project", + nextSummaryEndOfDayTicks: DateTime.UtcNow.AddDays(-2).Ticks); + + var futureProject = _projectData.GenerateProject(generateId: true, + organizationId: TestConstants.OrganizationId, name: "Future Project", + nextSummaryEndOfDayTicks: DateTime.UtcNow.AddDays(2).Ticks); + + await _repository.AddAsync([pastProject, futureProject], o => o.ImmediateConsistency()); + + // Act — with hourToSendNotificationsAfterUtcMidnight=0, threshold is current time + var results = await _repository.GetByNextSummaryNotificationOffsetAsync(0, limit: 50); + + // Assert — pastProject should be returned (ticks < threshold), futureProject should not + Assert.Contains(results.Documents, p => String.Equals(p.Id, pastProject.Id, StringComparison.Ordinal)); + Assert.DoesNotContain(results.Documents, p => String.Equals(p.Id, futureProject.Id, StringComparison.Ordinal)); + } } diff --git a/tests/Exceptionless.Tests/Repositories/StackRepositoryTests.cs b/tests/Exceptionless.Tests/Repositories/StackRepositoryTests.cs index cad325c3a5..54b4093187 100644 --- a/tests/Exceptionless.Tests/Repositories/StackRepositoryTests.cs +++ b/tests/Exceptionless.Tests/Repositories/StackRepositoryTests.cs @@ -8,6 +8,7 @@ using Foundatio.Caching; using Foundatio.Repositories; using Foundatio.Repositories.Options; +using Foundatio.Serializer; using Xunit; namespace Exceptionless.Tests.Repositories; @@ -15,12 +16,14 @@ namespace Exceptionless.Tests.Repositories; public sealed class StackRepositoryTests : IntegrationTestsBase { private readonly InMemoryCacheClient _cache; + private readonly ITextSerializer _serializer; private readonly StackData _stackData; private readonly IStackRepository _repository; public StackRepositoryTests(ITestOutputHelper output, AppWebHostFactory factory) : base(output, factory) { _cache = GetService() as InMemoryCacheClient ?? throw new InvalidOperationException(); + _serializer = GetService(); _stackData = GetService(); _repository = GetService(); } @@ -82,7 +85,7 @@ public async Task CanGetByStackHashAsync() Assert.Equal(misses, _cache.Misses); var result = await _repository.GetStackBySignatureHashAsync(stack.ProjectId, stack.SignatureHash); - Assert.Equal(stack.ToJson(), result.ToJson()); + JsonAssert.AssertJsonEquivalent(_serializer.SerializeToString(stack), _serializer.SerializeToString(result)); Assert.Equal(count + 2, _cache.Count); Assert.Equal(hits + 1, _cache.Hits); Assert.Equal(misses, _cache.Misses); diff --git a/tests/Exceptionless.Tests/Repositories/TokenRepositoryTests.cs b/tests/Exceptionless.Tests/Repositories/TokenRepositoryTests.cs index 33810e8e13..66744053c5 100644 --- a/tests/Exceptionless.Tests/Repositories/TokenRepositoryTests.cs +++ b/tests/Exceptionless.Tests/Repositories/TokenRepositoryTests.cs @@ -73,4 +73,26 @@ await _repository.AddAsync(new List { Assert.Equal(0, (await _repository.GetByTypeAndUserIdAsync(TokenType.Authentication, TestConstants.UserId)).Total); Assert.Equal(1, (await _repository.GetByOrganizationIdAsync(TestConstants.OrganizationId)).Total); } + + [Fact] + public async Task GetByTypeAndProjectId_FieldOr_MatchesProjectIdOrDefaultProjectId() + { + // Arrange + var utcNow = DateTime.UtcNow; + await _repository.AddAsync(new List + { + new() { OrganizationId = TestConstants.OrganizationId, ProjectId = TestConstants.ProjectId, Type = TokenType.Access, CreatedUtc = utcNow, UpdatedUtc = utcNow, Id = StringExtensions.GetNewToken() }, + new() { OrganizationId = TestConstants.OrganizationId, DefaultProjectId = TestConstants.ProjectId, Type = TokenType.Access, CreatedUtc = utcNow, UpdatedUtc = utcNow, Id = StringExtensions.GetNewToken() }, + new() { OrganizationId = TestConstants.OrganizationId, ProjectId = TestConstants.ProjectId, Type = TokenType.Authentication, CreatedUtc = utcNow, UpdatedUtc = utcNow, Id = StringExtensions.GetNewToken() }, + new() { OrganizationId = TestConstants.OrganizationId, ProjectId = TestConstants.ProjectIdWithNoRoles, Type = TokenType.Access, CreatedUtc = utcNow, UpdatedUtc = utcNow, Id = StringExtensions.GetNewToken() }, + }, o => o.ImmediateConsistency()); + + // Act + var accessResults = await _repository.GetByTypeAndProjectIdAsync(TokenType.Access, TestConstants.ProjectId, o => o.PageLimit(10)); + var authenticationResults = await _repository.GetByTypeAndProjectIdAsync(TokenType.Authentication, TestConstants.ProjectId, o => o.PageLimit(10)); + + // Assert + Assert.Equal(2, accessResults.Total); + Assert.Single(authenticationResults.Documents); + } } diff --git a/tests/Exceptionless.Tests/Search/EventStackFilterQueryVisitorTests.cs b/tests/Exceptionless.Tests/Search/EventStackFilterQueryVisitorTests.cs index f49f760dd2..908d412153 100644 --- a/tests/Exceptionless.Tests/Search/EventStackFilterQueryVisitorTests.cs +++ b/tests/Exceptionless.Tests/Search/EventStackFilterQueryVisitorTests.cs @@ -1,5 +1,5 @@ +using System.Text.Json; using Exceptionless.Core.Repositories.Queries; -using Newtonsoft.Json; using Xunit; using Xunit.Sdk; @@ -205,7 +205,7 @@ public override string ToString() public void Deserialize(IXunitSerializationInfo info) { string jsonValue = info.GetValue("objValue") ?? throw new InvalidOperationException("Missing objValue"); - var value = JsonConvert.DeserializeObject(jsonValue) ?? throw new InvalidOperationException("Failed to deserialize"); + var value = JsonSerializer.Deserialize(jsonValue) ?? throw new InvalidOperationException("Failed to deserialize"); Source = value.Source; Stack = value.Stack; InvertedStack = value.InvertedStack; @@ -214,7 +214,7 @@ public void Deserialize(IXunitSerializationInfo info) public void Serialize(IXunitSerializationInfo info) { - string? json = JsonConvert.SerializeObject(this); + string json = JsonSerializer.Serialize(this); info.AddValue("objValue", json); } } diff --git a/tests/Exceptionless.Tests/Serializer/CasingCompatibilityTests.cs b/tests/Exceptionless.Tests/Serializer/CasingCompatibilityTests.cs new file mode 100644 index 0000000000..6192483d61 --- /dev/null +++ b/tests/Exceptionless.Tests/Serializer/CasingCompatibilityTests.cs @@ -0,0 +1,438 @@ +using System.Collections.ObjectModel; +using System.Text.Json; +using Exceptionless.Core.Extensions; +using Exceptionless.Core.Jobs; +using Exceptionless.Core.Models; +using Exceptionless.Core.Models.Data; +using Exceptionless.Core.Queues.Models; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Serialization; +using Exceptionless.Core.Utility; +using Exceptionless.Tests.Extensions; +using Exceptionless.Tests.Utility; +using Foundatio.Queues; +using Foundatio.Repositories; +using Foundatio.Serializer; +using Xunit; + +namespace Exceptionless.Tests.Serializer; + +/// +/// Tests verifying that events submitted with PascalCase and camelCase property names +/// are deserialized and processed correctly — matching behavior from main branch (Newtonsoft). +/// These tests reproduce issues identified in the serialization audit diff. +/// +public class CasingCompatibilityTests : TestWithServices +{ + private readonly ITextSerializer _serializer; + private readonly JsonSerializerOptions _jsonOptions; + + public CasingCompatibilityTests(ITestOutputHelper output) : base(output) + { + _serializer = GetService(); + _jsonOptions = GetService(); + } + + [Theory] + [InlineData("""{"type":"error"}""", "error")] + [InlineData("""{"Type":"error"}""", "error")] + [InlineData("""{"TYPE":"error"}""", "error")] + public void Deserialize_TypeProperty_MatchesAllCasings(string json, string expected) + { + // Arrange + + // Act + var ev = JsonSerializer.Deserialize(json, _jsonOptions); + + // Assert + Assert.NotNull(ev); + Assert.Equal(expected, ev.Type); + } + + [Theory] + [InlineData("""{"message":"hello"}""", "hello")] + [InlineData("""{"Message":"hello"}""", "hello")] + [InlineData("""{"MESSAGE":"hello"}""", "hello")] + public void Deserialize_MessageProperty_MatchesAllCasings(string json, string expected) + { + // Arrange + + // Act + var ev = JsonSerializer.Deserialize(json, _jsonOptions); + + // Assert + Assert.NotNull(ev); + Assert.Equal(expected, ev.Message); + } + + [Theory] + [InlineData("""{"value":42.5}""", 42.5)] + [InlineData("""{"Value":42.5}""", 42.5)] + [InlineData("""{"VALUE":42.5}""", 42.5)] + public void Deserialize_ValueProperty_MatchesAllCasings(string json, decimal expected) + { + // Arrange + + // Act + var ev = JsonSerializer.Deserialize(json, _jsonOptions); + + // Assert + Assert.NotNull(ev); + Assert.Equal(expected, ev.Value); + } + + [Theory] + [InlineData("""{"count":5}""", 5)] + [InlineData("""{"Count":5}""", 5)] + [InlineData("""{"COUNT":5}""", 5)] + public void Deserialize_CountProperty_MatchesAllCasings(string json, int expected) + { + // Arrange + + // Act + var ev = JsonSerializer.Deserialize(json, _jsonOptions); + + // Assert + Assert.NotNull(ev); + Assert.Equal(expected, ev.Count); + } + + [Theory] + [InlineData("""{"tags":["a","b"]}""")] + [InlineData("""{"Tags":["a","b"]}""")] + [InlineData("""{"TAGS":["a","b"]}""")] + public void Deserialize_TagsProperty_MatchesAllCasings(string json) + { + // Arrange + + // Act + var ev = JsonSerializer.Deserialize(json, _jsonOptions); + + // Assert + Assert.NotNull(ev); + Assert.NotNull(ev.Tags); + Assert.Equal(2, ev.Tags.Count); + Assert.Contains("a", ev.Tags); + Assert.Contains("b", ev.Tags); + } + + [Theory] + [InlineData("""{"geo":"40.7,-74.0"}""", "40.7,-74.0")] + [InlineData("""{"Geo":"40.7,-74.0"}""", "40.7,-74.0")] + [InlineData("""{"GEO":"40.7,-74.0"}""", "40.7,-74.0")] + public void Deserialize_GeoProperty_MatchesAllCasings(string json, string expected) + { + // Arrange + + // Act + var ev = JsonSerializer.Deserialize(json, _jsonOptions); + + // Assert + Assert.NotNull(ev); + Assert.Equal(expected, ev.Geo); + } + + [Theory] + [InlineData("""{"reference_id":"abc"}""", "abc")] + [InlineData("""{"REFERENCE_ID":"abc"}""", "abc")] + [InlineData("""{"referenceId":"abc"}""", "abc")] + [InlineData("""{"ReferenceId":"abc"}""", "abc")] + public void Deserialize_ReferenceId_MatchesAllCasings(string json, string expected) + { + // Arrange + + // Act + var ev = JsonSerializer.Deserialize(json, _jsonOptions); + + // Assert + Assert.NotNull(ev); + Assert.Equal(expected, ev.ReferenceId); + } + + [Fact] + public void Deserialize_PascalCaseFullEvent_AllPropertiesBound() + { + // Arrange + /* language=json */ + const string json = """ + { + "Type": "error", + "Message": "Test error with PascalCase", + "Tags": ["audit", "PascalCase"], + "ReferenceId": "pascal-001", + "Count": 1, + "Value": 42.5, + "Geo": "40.7128,-74.0060", + "Date": "2026-05-20T12:00:00+00:00" + } + """; + + // Act + var ev = JsonSerializer.Deserialize(json, _jsonOptions); + + // Assert + Assert.NotNull(ev); + Assert.Equal("error", ev.Type); + Assert.Equal("Test error with PascalCase", ev.Message); + Assert.Equal("pascal-001", ev.ReferenceId); + Assert.Equal(1, ev.Count); + Assert.Equal(42.5m, ev.Value); + Assert.Equal("40.7128,-74.0060", ev.Geo); + Assert.NotNull(ev.Tags); + Assert.Equal(2, ev.Tags.Count); + Assert.Contains("audit", ev.Tags); + Assert.Contains("PascalCase", ev.Tags); + } + + [Fact] + public void Deserialize_CamelCaseFullEvent_AllPropertiesBound() + { + // Arrange + /* language=json */ + const string json = """ + { + "type": "error", + "message": "Test error with camelCase", + "tags": ["audit", "camelCase"], + "referenceId": "camel-001", + "count": 1, + "value": 42.5, + "geo": "40.7128,-74.0060", + "date": "2026-05-20T12:00:00+00:00" + } + """; + + // Act + var ev = JsonSerializer.Deserialize(json, _jsonOptions); + + // Assert + Assert.NotNull(ev); + Assert.Equal("error", ev.Type); + Assert.Equal("Test error with camelCase", ev.Message); + Assert.Equal("camel-001", ev.ReferenceId); + Assert.Equal(1, ev.Count); + Assert.Equal(42.5m, ev.Value); + Assert.Equal("40.7128,-74.0060", ev.Geo); + Assert.NotNull(ev.Tags); + Assert.Equal(2, ev.Tags.Count); + } + + [Fact] + public void Deserialize_DateOnlyString_PreservedAsString() + { + // Arrange + /* language=json */ + const string json = """ + { + "type": "log", + "data": { + "date_only": "2026-01-15", + "not_a_date": "2026-13-45T99:99:99Z", + "iso_utc": "2026-05-20T12:00:00Z" + } + } + """; + + // Act + var ev = JsonSerializer.Deserialize(json, _jsonOptions); + + // Assert + Assert.NotNull(ev); + Assert.NotNull(ev.Data); + + var dateOnly = ev.Data["date_only"]; + Assert.IsType(dateOnly); + Assert.Equal("2026-01-15", dateOnly); + + // Invalid date strings should always stay as strings + var notADate = ev.Data["not_a_date"]; + Assert.IsType(notADate); + Assert.Equal("2026-13-45T99:99:99Z", notADate); + + // Full ISO dates ARE expected to parse to DateTimeOffset + var isoUtc = ev.Data["iso_utc"]; + Assert.IsType(isoUtc); + } + + [Fact] + public void Deserialize_NumericZeroVsZeroPointZero_Preserved() + { + // Arrange + /* language=json */ + const string json = """ + { + "type": "log", + "data": { + "zero_int": 0, + "zero_float": 0.0, + "one_point_zero": 1.0, + "one_int": 1 + } + } + """; + + // Act + var ev = JsonSerializer.Deserialize(json, _jsonOptions); + + // Assert + Assert.NotNull(ev); + Assert.NotNull(ev.Data); + + // Integer 0 should be int (or long in ES mode) + var zeroInt = ev.Data["zero_int"]; + Assert.IsType(zeroInt); + Assert.Equal(0, zeroInt); + + // 0.0 should be decimal (floating-point preserved) + var zeroFloat = ev.Data["zero_float"]; + Assert.IsType(zeroFloat); + Assert.Equal(0.0m, zeroFloat); + + // 1.0 should be decimal + var oneFloat = ev.Data["one_point_zero"]; + Assert.IsType(oneFloat); + Assert.Equal(1.0m, oneFloat); + + // 1 should be int + var oneInt = ev.Data["one_int"]; + Assert.IsType(oneInt); + Assert.Equal(1, oneInt); + } + + [Fact] + public void Serialize_EventValueZero_SerializesAsInteger() + { + // Arrange + var ev = new Event { Type = "log", Value = 0m }; + + // Act + string json = JsonSerializer.Serialize(ev, _jsonOptions); + + // Assert + var doc = JsonDocument.Parse(json); + var valueElement = doc.RootElement.GetProperty("value"); + Assert.Equal("0", valueElement.GetRawText()); + } + + [Fact] + public void Serialize_EmptyTags_OmittedFromOutput() + { + // Arrange + var ev = new Event { Type = "log", Tags = [] }; + + // Act + string json = JsonSerializer.Serialize(ev, _jsonOptions); + + // Assert + var doc = JsonDocument.Parse(json); + Assert.False(doc.RootElement.TryGetProperty("tags", out _)); + } + + [Fact] + public void Serialize_EmptyData_OmittedFromOutput() + { + // Arrange + var ev = new Event { Type = "log", Data = new DataDictionary() }; + + // Act + string json = JsonSerializer.Serialize(ev, _jsonOptions); + + // Assert + var doc = JsonDocument.Parse(json); + Assert.False(doc.RootElement.TryGetProperty("data", out _)); + } + + [Fact] + public void Serialize_NonEmptyTags_IncludedInOutput() + { + // Arrange + var ev = new Event { Type = "log", Tags = ["tag1"] }; + + // Act + string json = JsonSerializer.Serialize(ev, _jsonOptions); + + // Assert + var doc = JsonDocument.Parse(json); + Assert.True(doc.RootElement.TryGetProperty("tags", out var tagsEl)); + Assert.Equal(1, tagsEl.GetArrayLength()); + } + + [Fact] + public void Serialize_EmptyReferences_OmittedFromOutput() + { + // Arrange + var stack = new Stack { OrganizationId = "org1", ProjectId = "proj1", SignatureHash = "abc", Type = "error", Title = "test" }; + stack.References = new Collection(); + + // Act + string json = JsonSerializer.Serialize(stack, _jsonOptions); + + // Assert + var doc = JsonDocument.Parse(json); + Assert.False(doc.RootElement.TryGetProperty("references", out _)); + } + + [Fact] + public void Serialize_NonEmptyReferences_IncludedInOutput() + { + // Arrange + var stack = new Stack { OrganizationId = "org1", ProjectId = "proj1", SignatureHash = "abc", Type = "error", Title = "test" }; + stack.References = new Collection(["ref1"]); + + // Act + string json = JsonSerializer.Serialize(stack, _jsonOptions); + + // Assert + var doc = JsonDocument.Parse(json); + Assert.True(doc.RootElement.TryGetProperty("references", out var refsEl)); + Assert.Equal(1, refsEl.GetArrayLength()); + } + + [Fact] + public void Deserialize_MissingTags_DefaultsToEmpty() + { + // Arrange + const string json = """{"type":"error","source":"test"}"""; + + // Act + var ev = JsonSerializer.Deserialize(json, _jsonOptions); + + // Assert + Assert.NotNull(ev); + Assert.NotNull(ev.Tags); + Assert.Empty(ev.Tags); + } + + [Fact] + public void Deserialize_MissingReferences_DefaultsToEmpty() + { + // Arrange + const string json = """{"organization_id":"org1","project_id":"proj1","signature_hash":"abc","type":"error","title":"test"}"""; + + // Act + var stack = JsonSerializer.Deserialize(json, _jsonOptions); + + // Assert + Assert.NotNull(stack); + Assert.NotNull(stack.References); + Assert.Empty(stack.References); + } + + [Fact] + public void Roundtrip_EmptyCollections_PreservedOnDeserialize() + { + // Arrange + var original = new Event { Type = "log", Tags = [], Data = new DataDictionary() }; + + // Act + string json = JsonSerializer.Serialize(original, _jsonOptions); + var deserialized = JsonSerializer.Deserialize(json, _jsonOptions); + + // Assert + Assert.DoesNotContain("tags", json); + Assert.DoesNotContain("data", json); + Assert.NotNull(deserialized); + Assert.NotNull(deserialized.Tags); + Assert.Empty(deserialized.Tags); + } +} diff --git a/tests/Exceptionless.Tests/Serializer/Data/event-with-unknown-properties-roundtrip.expected.json b/tests/Exceptionless.Tests/Serializer/Data/event-with-unknown-properties-roundtrip.expected.json new file mode 100644 index 0000000000..c16a43f2d8 --- /dev/null +++ b/tests/Exceptionless.Tests/Serializer/Data/event-with-unknown-properties-roundtrip.expected.json @@ -0,0 +1,18 @@ +{ + "date": "0001-01-01T00:00:00+00:00", + "tags": [ + "One", + "Two" + ], + "message": "Hello", + "data": { + "SomeString": "Hi", + "SomeBool": false, + "SomeNum": 1, + "UnknownProp": { + "Blah": "SomeVal" + }, + "UnknownSerializedProp": "{\"Blah\":\"SomeVal\"}" + }, + "reference_id": "12" +} diff --git a/tests/Exceptionless.Tests/Serializer/Models/DataDictionarySerializerTests.cs b/tests/Exceptionless.Tests/Serializer/Models/DataDictionarySerializerTests.cs index dae967ec08..20ea71665f 100644 --- a/tests/Exceptionless.Tests/Serializer/Models/DataDictionarySerializerTests.cs +++ b/tests/Exceptionless.Tests/Serializer/Models/DataDictionarySerializerTests.cs @@ -1,6 +1,5 @@ using Exceptionless.Core.Models; using Foundatio.Serializer; -using Newtonsoft.Json.Linq; using Xunit; namespace Exceptionless.Tests.Serializer.Models; @@ -56,27 +55,26 @@ public void Deserialize_EmptyDictionary_ReturnsEmptyData() } /// - /// Reproduces production bug where JObject/JArray values in DataDictionary - /// (stored by Newtonsoft-based DataObjectConverter when reading from Elasticsearch) - /// serialize as nested empty arrays instead of proper JSON when written by STJ. + /// Verifies Dictionary values in DataDictionary (from ObjectToInferredTypesConverter + /// when reading from Elasticsearch) serialize correctly to JSON. /// [Fact] - public void Serialize_JObjectValue_WritesCorrectJson() + public void Serialize_DictionaryValue_WritesCorrectJson() { - // Arrange — simulate Elasticsearch read path storing JObject in DataDictionary - var jObject = JObject.Parse(""" + // Arrange — simulate Elasticsearch read path storing Dictionary in DataDictionary + var dict = new Dictionary + { + ["docsSecondari"] = new List { - "docsSecondari": [ - { "tipo": "CI", "numero": "AB123" }, - { "tipo": "PP", "numero": "CD456" } - ], - "docPrimario": { "tipo": "DL", "numero": "XY789" }, - "numeroDocumentiSecondari": 2, - "AlreadyImported": true - } - """); - - var data = new DataDictionary { ["TestUfficialeVO"] = jObject }; + new Dictionary { ["tipo"] = "CI", ["numero"] = "AB123" }, + new Dictionary { ["tipo"] = "PP", ["numero"] = "CD456" } + }, + ["docPrimario"] = new Dictionary { ["tipo"] = "DL", ["numero"] = "XY789" }, + ["numeroDocumentiSecondari"] = 2, + ["AlreadyImported"] = true + }; + + var data = new DataDictionary { ["TestUfficialeVO"] = dict }; // Act string json = _serializer.SerializeToString(data); @@ -91,14 +89,14 @@ public void Serialize_JObjectValue_WritesCorrectJson() } /// - /// Verifies JArray values in DataDictionary serialize correctly. + /// Verifies List values in DataDictionary serialize correctly. /// [Fact] - public void Serialize_JArrayValue_WritesCorrectJson() + public void Serialize_ListValue_WritesCorrectJson() { - // Arrange — simulate Elasticsearch storing JArray in DataDictionary - var jArray = JArray.Parse("""["tag1", "tag2", "tag3"]"""); - var data = new DataDictionary { ["Tags"] = jArray }; + // Arrange — simulate Elasticsearch storing List in DataDictionary + var list = new List { "tag1", "tag2", "tag3" }; + var data = new DataDictionary { ["Tags"] = list }; // Act string json = _serializer.SerializeToString(data); @@ -111,29 +109,31 @@ public void Serialize_JArrayValue_WritesCorrectJson() } /// - /// Verifies deeply nested JObject structures serialize correctly, - /// matching the exact production data pattern that was broken. + /// Verifies deeply nested Dictionary structures serialize correctly, + /// matching the exact production data pattern. /// [Fact] - public void Serialize_DeeplyNestedJObject_PreservesStructure() + public void Serialize_DeeplyNestedDictionary_PreservesStructure() { // Arrange — nested structure matching production data shape - var jObject = JObject.Parse(""" + var dict = new Dictionary + { + ["items"] = new List { - "items": [ + new Dictionary + { + ["name"] = "item1", + ["children"] = new List { - "name": "item1", - "children": [ - { "id": 1, "value": "a" }, - { "id": 2, "value": "b" } - ] + new Dictionary { ["id"] = 1, ["value"] = "a" }, + new Dictionary { ["id"] = 2, ["value"] = "b" } } - ], - "count": 1 - } - """); + } + }, + ["count"] = 1 + }; - var data = new DataDictionary { ["NestedData"] = jObject }; + var data = new DataDictionary { ["NestedData"] = dict }; // Act string json = _serializer.SerializeToString(data); diff --git a/tests/Exceptionless.Tests/Serializer/Models/DataDictionaryTests.cs b/tests/Exceptionless.Tests/Serializer/Models/DataDictionaryTests.cs index cc91b0837c..8c1b7d5722 100644 --- a/tests/Exceptionless.Tests/Serializer/Models/DataDictionaryTests.cs +++ b/tests/Exceptionless.Tests/Serializer/Models/DataDictionaryTests.cs @@ -1,26 +1,22 @@ -using System.Text.Json; using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Models.Data; using Foundatio.Serializer; -using Newtonsoft.Json.Linq; using Xunit; namespace Exceptionless.Tests.Serializer.Models; /// /// Tests for DataDictionary.GetValue extension method. -/// Verifies deserialization from typed objects, JObject (Elasticsearch), JSON strings, and round-trips. +/// Verifies deserialization from typed objects, Dictionary (Elasticsearch), JSON strings, and round-trips. /// public class DataDictionaryTests : TestWithServices { private readonly ITextSerializer _serializer; - private readonly JsonSerializerOptions _jsonOptions; public DataDictionaryTests(ITestOutputHelper output) : base(output) { _serializer = GetService(); - _jsonOptions = GetService(); } [Fact] @@ -31,7 +27,7 @@ public void GetValue_DirectUserInfoType_ReturnsTypedValue() var data = new DataDictionary { { "user", userInfo } }; // Act - var result = data.GetValue("user", _jsonOptions); + var result = data.GetValue("user", _serializer); // Assert Assert.NotNull(result); @@ -46,7 +42,7 @@ public void GetValue_DirectStringType_ReturnsStringValue() var data = new DataDictionary { { "version", "1.0.0" } }; // Act - string? result = data.GetValue("version", _jsonOptions); + string? result = data.GetValue("version", _serializer); // Assert Assert.Equal("1.0.0", result); @@ -59,45 +55,45 @@ public void GetValue_DirectIntType_ReturnsIntValue() var data = new DataDictionary { { "count", 42 } }; // Act - int result = data.GetValue("count", _jsonOptions); + int result = data.GetValue("count", _serializer); // Assert Assert.Equal(42, result); } [Fact] - public void GetValue_JObjectWithUserInfo_ReturnsTypedUserInfo() + public void GetValue_DictionaryWithUserInfo_ReturnsTypedUserInfo() { - // Arrange - JObject comes from Elasticsearch via NEST/JSON.NET - var jObject = JObject.FromObject(new { Identity = "jobj@test.com", Name = "JObject User" }); - var data = new DataDictionary { { "user", jObject } }; + // Arrange - Dictionary comes from Elasticsearch via new Elastic client + ObjectToInferredTypesConverter + var dict = new Dictionary { ["identity"] = "dict@test.com", ["name"] = "Dict User" }; + var data = new DataDictionary { { "user", dict } }; // Act - var result = data.GetValue("user", _jsonOptions); + var result = data.GetValue("user", _serializer); // Assert Assert.NotNull(result); - Assert.Equal("jobj@test.com", result.Identity); - Assert.Equal("JObject User", result.Name); + Assert.Equal("dict@test.com", result.Identity); + Assert.Equal("Dict User", result.Name); } [Fact] - public void GetValue_JObjectWithError_ReturnsTypedError() + public void GetValue_DictionaryWithError_ReturnsTypedError() { - // Arrange - var jObject = JObject.FromObject(new + // Arrange - simulates ObjectToInferredTypesConverter output (snake_case keys from ES) + var dict = new Dictionary { - Message = "Test error", - Type = "System.Exception", - StackTrace = new[] + ["message"] = "Test error", + ["type"] = "System.Exception", + ["stack_trace"] = new List { - new { Name = "TestMethod", DeclaringNamespace = "Tests", DeclaringType = "TestClass" } + new Dictionary { ["name"] = "TestMethod", ["declaring_namespace"] = "Tests", ["declaring_type"] = "TestClass" } } - }); - var data = new DataDictionary { { "@error", jObject } }; + }; + var data = new DataDictionary { { "@error", dict } }; // Act - var result = data.GetValue("@error", _jsonOptions); + var result = data.GetValue("@error", _serializer); // Assert Assert.NotNull(result); @@ -108,22 +104,22 @@ public void GetValue_JObjectWithError_ReturnsTypedError() } [Fact] - public void GetValue_JObjectWithRequestInfo_ReturnsTypedRequestInfo() + public void GetValue_DictionaryWithRequestInfo_ReturnsTypedRequestInfo() { // Arrange - var jObject = JObject.FromObject(new + var dict = new Dictionary { - HttpMethod = "GET", - Path = "/api/test", - Host = "localhost", - Port = 443, - IsSecure = true, - ClientIpAddress = "127.0.0.1" - }); - var data = new DataDictionary { { "@request", jObject } }; + ["http_method"] = "GET", + ["path"] = "/api/test", + ["host"] = "localhost", + ["port"] = 443, + ["is_secure"] = true, + ["client_ip_address"] = "127.0.0.1" + }; + var data = new DataDictionary { { "@request", dict } }; // Act - var result = data.GetValue("@request", _jsonOptions); + var result = data.GetValue("@request", _serializer); // Assert Assert.NotNull(result); @@ -135,54 +131,108 @@ public void GetValue_JObjectWithRequestInfo_ReturnsTypedRequestInfo() } [Fact] - public void GetValue_JObjectWithEnvironmentInfo_ReturnsTypedEnvironmentInfo() + public void GetValue_DictionaryWithPascalCaseRequestInfo_ReturnsTypedRequestInfo() { // Arrange - var jObject = JObject.FromObject(new + var dict = new Dictionary { - MachineName = "TEST-MACHINE", - ProcessorCount = 8, - TotalPhysicalMemory = 16000000000L, - OSName = "Windows", - OSVersion = "10.0" - }); - var data = new DataDictionary { { "@environment", jObject } }; + ["HttpMethod"] = "POST", + ["Path"] = "/api/pascal", + ["Host"] = "localhost", + ["Port"] = 8443, + ["IsSecure"] = true, + ["ClientIpAddress"] = "127.0.0.2" + }; + var data = new DataDictionary { { "@request", dict } }; + + // Act + var result = data.GetValue("@request", _serializer); + + // Assert + Assert.NotNull(result); + Assert.Equal("POST", result.HttpMethod); + Assert.Equal("/api/pascal", result.Path); + Assert.Equal("localhost", result.Host); + Assert.Equal(8443, result.Port); + Assert.True(result.IsSecure); + Assert.Equal("127.0.0.2", result.ClientIpAddress); + } + + [Fact] + public void GetValue_DictionaryWithEnvironmentInfo_ReturnsTypedEnvironmentInfo() + { + // Arrange + var dict = new Dictionary + { + ["machine_name"] = "TEST-MACHINE", + ["processor_count"] = 8, + ["total_physical_memory"] = 16000000000L, + ["o_s_name"] = "Windows", + ["o_s_version"] = "10.0" + }; + var data = new DataDictionary { { "@environment", dict } }; // Act - var result = data.GetValue("@environment", _jsonOptions); + var result = data.GetValue("@environment", _serializer); // Assert Assert.NotNull(result); Assert.Equal("TEST-MACHINE", result.MachineName); Assert.Equal(8, result.ProcessorCount); + Assert.Equal("Windows", result.OSName); + Assert.Equal("10.0", result.OSVersion); } [Fact] - public void GetValue_JObjectWithNestedError_ReturnsNestedHierarchy() + public void GetValue_DictionaryWithPascalCaseEnvironmentInfo_ReturnsTypedEnvironmentInfo() { // Arrange - /* language=json */ - const string jsonInput = """ + var dict = new Dictionary + { + ["MachineName"] = "PASCAL-MACHINE", + ["ProcessorCount"] = 16, + ["TotalPhysicalMemory"] = 32000000000L, + ["OSName"] = "Windows", + ["OSVersion"] = "11.0" + }; + var data = new DataDictionary { { "@environment", dict } }; + + // Act + var result = data.GetValue("@environment", _serializer); + + // Assert + Assert.NotNull(result); + Assert.Equal("PASCAL-MACHINE", result.MachineName); + Assert.Equal(16, result.ProcessorCount); + Assert.Equal(32000000000L, result.TotalPhysicalMemory); + Assert.Equal("Windows", result.OSName); + Assert.Equal("11.0", result.OSVersion); + } + + [Fact] + public void GetValue_DictionaryWithNestedError_ReturnsNestedHierarchy() + { + // Arrange - simulates nested object from ObjectToInferredTypesConverter + var dict = new Dictionary { - "Message": "Outer JObject error", - "Type": "OuterException", - "Inner": { - "Message": "Inner JObject error", - "Type": "InnerException" + ["message"] = "Outer error", + ["type"] = "OuterException", + ["inner"] = new Dictionary + { + ["message"] = "Inner error", + ["type"] = "InnerException" } - } - """; - var jObject = JObject.Parse(jsonInput); - var data = new DataDictionary { { "@error", jObject } }; + }; + var data = new DataDictionary { { "@error", dict } }; // Act - var result = data.GetValue("@error", _jsonOptions); + var result = data.GetValue("@error", _serializer); // Assert Assert.NotNull(result); - Assert.Equal("Outer JObject error", result.Message); + Assert.Equal("Outer error", result.Message); Assert.NotNull(result.Inner); - Assert.Equal("Inner JObject error", result.Inner.Message); + Assert.Equal("Inner error", result.Inner.Message); } [Fact] @@ -194,7 +244,7 @@ public void GetValue_JsonStringWithUserInfo_ReturnsTypedUserInfo() var data = new DataDictionary { { "user", json } }; // Act - var result = data.GetValue("user", _jsonOptions); + var result = data.GetValue("user", _serializer); // Assert Assert.NotNull(result); @@ -211,7 +261,7 @@ public void GetValue_JsonStringWithError_ReturnsTypedError() var data = new DataDictionary { { "@error", json } }; // Act - var result = data.GetValue("@error", _jsonOptions); + var result = data.GetValue("@error", _serializer); // Assert Assert.NotNull(result); @@ -228,7 +278,7 @@ public void GetValue_JsonStringWithRequestInfo_ReturnsTypedRequestInfo() var data = new DataDictionary { { "@request", json } }; // Act - var result = data.GetValue("@request", _jsonOptions); + var result = data.GetValue("@request", _serializer); // Assert Assert.NotNull(result); @@ -245,7 +295,7 @@ public void GetValue_JsonStringWithEnvironmentInfo_ReturnsTypedEnvironmentInfo() var data = new DataDictionary { { "@environment", json } }; // Act - var result = data.GetValue("@environment", _jsonOptions); + var result = data.GetValue("@environment", _serializer); // Assert Assert.NotNull(result); @@ -262,7 +312,7 @@ public void GetValue_JsonStringWithSimpleError_ReturnsTypedSimpleError() var data = new DataDictionary { { "@simple_error", json } }; // Act - var result = data.GetValue("@simple_error", _jsonOptions); + var result = data.GetValue("@simple_error", _serializer); // Assert Assert.NotNull(result); @@ -279,7 +329,7 @@ public void GetValue_JsonStringWithNestedError_ReturnsNestedHierarchy() var data = new DataDictionary { { "@error", json } }; // Act - var result = data.GetValue("@error", _jsonOptions); + var result = data.GetValue("@error", _serializer); // Assert Assert.NotNull(result); @@ -295,7 +345,7 @@ public void GetValue_NonJsonString_ReturnsNull() var data = new DataDictionary { { "text", "not json" } }; // Act - var result = data.GetValue("text", _jsonOptions); + var result = data.GetValue("text", _serializer); // Assert Assert.Null(result); @@ -308,7 +358,7 @@ public void GetValue_MissingKey_ThrowsKeyNotFoundException() var data = new DataDictionary(); // Act & Assert - Assert.Throws(() => data.GetValue("nonexistent", _jsonOptions)); + Assert.Throws(() => data.GetValue("nonexistent", _serializer)); } [Fact] @@ -318,7 +368,7 @@ public void GetValue_NullValue_ReturnsNull() var data = new DataDictionary { { "nullable", null! } }; // Act - var result = data.GetValue("nullable", _jsonOptions); + var result = data.GetValue("nullable", _serializer); // Assert Assert.Null(result); @@ -331,7 +381,7 @@ public void GetValue_IncompatibleType_ReturnsNull() var data = new DataDictionary { { "number", 42 } }; // Act - var result = data.GetValue("number", _jsonOptions); + var result = data.GetValue("number", _serializer); // Assert Assert.Null(result); @@ -346,7 +396,7 @@ public void GetValue_MalformedJsonString_ReturnsDefaultProperties() var data = new DataDictionary { { "user", json } }; // Act - var result = data.GetValue("user", _jsonOptions); + var result = data.GetValue("user", _serializer); // Assert Assert.NotNull(result); @@ -369,7 +419,7 @@ public void Deserialize_DataDictionaryWithUserInfoAfterRoundTrip_PreservesTypedD // Assert Assert.NotNull(deserialized); Assert.True(deserialized.ContainsKey("@user")); - var userInfo = deserialized.GetValue("@user", _jsonOptions); + var userInfo = deserialized.GetValue("@user", _serializer); Assert.NotNull(userInfo); Assert.Equal("user@test.com", userInfo.Identity); Assert.Equal("Test User", userInfo.Name); @@ -429,7 +479,7 @@ public void Deserialize_UserInfoAfterRoundTrip_PreservesAllProperties() // Assert Assert.NotNull(deserialized); - var result = deserialized.GetValue("@user", _jsonOptions); + var result = deserialized.GetValue("@user", _serializer); Assert.NotNull(result); Assert.Equal("stj@test.com", result.Identity); Assert.Equal("STJ Test User", result.Name); @@ -463,7 +513,7 @@ public void Deserialize_ErrorAfterRoundTrip_PreservesComplexStructure() // Assert Assert.NotNull(deserialized); - var result = deserialized.GetValue("@error", _jsonOptions); + var result = deserialized.GetValue("@error", _serializer); Assert.NotNull(result); Assert.Equal("Test Exception", result.Message); Assert.Equal("System.InvalidOperationException", result.Type); @@ -495,7 +545,7 @@ public void Deserialize_RequestInfoAfterRoundTrip_PreservesAllProperties() // Assert Assert.NotNull(deserialized); - var result = deserialized.GetValue("@request", _jsonOptions); + var result = deserialized.GetValue("@request", _serializer); Assert.NotNull(result); Assert.Equal("POST", result.HttpMethod); Assert.Equal("/api/events", result.Path); @@ -525,7 +575,7 @@ public void Deserialize_EnvironmentInfoAfterRoundTrip_PreservesAllProperties() // Assert Assert.NotNull(deserialized); - var result = deserialized.GetValue("@environment", _jsonOptions); + var result = deserialized.GetValue("@environment", _serializer); Assert.NotNull(result); Assert.Equal("TEST-MACHINE", result.MachineName); Assert.Equal(16, result.ProcessorCount); @@ -555,7 +605,7 @@ public void Deserialize_NestedErrorAfterRoundTrip_PreservesInnerError() // Assert Assert.NotNull(deserialized); - var result = deserialized.GetValue("@error", _jsonOptions); + var result = deserialized.GetValue("@error", _serializer); Assert.NotNull(result); Assert.Equal("Outer exception", result.Message); Assert.NotNull(result.Inner); @@ -582,7 +632,7 @@ public void Deserialize_MixedDataTypesAfterRoundTrip_PreservesAllTypes() // Assert Assert.NotNull(deserialized); - var userInfo = deserialized.GetValue("@user", _jsonOptions); + var userInfo = deserialized.GetValue("@user", _serializer); Assert.NotNull(userInfo); Assert.Equal("user@test.com", userInfo.Identity); @@ -611,7 +661,7 @@ public void Deserialize_NestedDataDictionaryAfterRoundTrip_PreservesNestedData() // Assert Assert.NotNull(deserialized); - var result = deserialized.GetValue("@user", _jsonOptions); + var result = deserialized.GetValue("@user", _serializer); Assert.NotNull(result); Assert.Equal("user@test.com", result.Identity); Assert.NotNull(result.Data); @@ -631,7 +681,7 @@ public void GetValue_DictionaryOfStringObject_DeserializesToTypedObject() var data = new DataDictionary { { "@user", dictionary } }; // Act - var result = data.GetValue("@user", _jsonOptions); + var result = data.GetValue("@user", _serializer); // Assert Assert.NotNull(result); @@ -651,7 +701,7 @@ public void GetValue_ListOfObjects_DeserializesToTypedCollection() var data = new DataDictionary { { "frames", list } }; // Act - var result = data.GetValue>("frames", _jsonOptions); + var result = data.GetValue>("frames", _serializer); // Assert Assert.NotNull(result); @@ -661,4 +711,241 @@ public void GetValue_ListOfObjects_DeserializesToTypedCollection() Assert.Equal("Frame2", result[1].Name); Assert.Equal(20, result[1].LineNumber); } + + [Fact] + public void GetValue_SnakeCaseJsonString_DeserializesViaPrimarySerializer() + { + // Arrange — current-format snake_case data written by STJ + var data = new DataDictionary + { + { Event.KnownDataKeys.EnvironmentInfo, """{"machine_name":"PROD-01","processor_count":8,"total_physical_memory":16384,"command_line":"dotnet run"}""" } + }; + + // Act + var result = data.GetValue(Event.KnownDataKeys.EnvironmentInfo, _serializer); + + // Assert — primary serializer handles snake_case correctly + Assert.NotNull(result); + Assert.Equal("PROD-01", result.MachineName); + Assert.Equal(8, result.ProcessorCount); + Assert.Equal(16384, result.TotalPhysicalMemory); + Assert.Equal("dotnet run", result.CommandLine); + } + + [Fact] + public void GetValue_SnakeCaseDictionary_DeserializesViaPrimarySerializer() + { + // Arrange + var dict = new Dictionary + { + { "machine_name", "DICT-01" }, + { "processor_count", 16L }, + { "total_physical_memory", 32768L }, + { "command_line", "app.exe --verbose" } + }; + var data = new DataDictionary { { Event.KnownDataKeys.EnvironmentInfo, dict } }; + + // Act + var result = data.GetValue(Event.KnownDataKeys.EnvironmentInfo, _serializer); + + // Assert + Assert.NotNull(result); + Assert.Equal("DICT-01", result.MachineName); + Assert.Equal(16, result.ProcessorCount); + Assert.Equal(32768, result.TotalPhysicalMemory); + Assert.Equal("app.exe --verbose", result.CommandLine); + } + + // --- Legacy PascalCase bridge (V1 client / pre-STJ data) --------------------------------- + // STJ's PropertyNameCaseInsensitive + SnakeCaseLower naming policy only handles case + // differences ("Message" ↔ "message"). It cannot structurally match multi-word PascalCase + // ("ClientIpAddress") against snake_case ("client_ip_address"). GetValue normalizes + // typed-property keys recursively to bridge the two formats while preserving + // user-provided dictionary keys (Error.Data, QueryString, etc.) exactly as submitted. + + [Fact] + public void GetValue_PascalCaseDictionaryWithRequestInfo_MapsMultiWordKeys() + { + // Arrange — simulates legacy V1 submission stored before STJ migration. + var dict = new Dictionary + { + ["HttpMethod"] = "GET", + ["ClientIpAddress"] = "10.0.0.1", + ["IsSecure"] = true, + ["UserAgent"] = "Test/1.0", + ["Path"] = "/api/test" + }; + var data = new DataDictionary { { "@request", dict } }; + + // Act + var result = data.GetValue("@request", _serializer); + + // Assert + Assert.NotNull(result); + Assert.Equal("GET", result.HttpMethod); + Assert.Equal("10.0.0.1", result.ClientIpAddress); + Assert.True(result.IsSecure); + Assert.Equal("Test/1.0", result.UserAgent); + Assert.Equal("/api/test", result.Path); + } + + [Fact] + public void GetValue_PascalCaseDictionaryWithEnvironmentInfo_MapsMultiWordKeys() + { + // Arrange + var dict = new Dictionary + { + ["MachineName"] = "LEGACY-MACHINE", + ["ProcessorCount"] = 4, + ["TotalPhysicalMemory"] = 8000000000L, + ["OSName"] = "Windows", + ["OSVersion"] = "10.0", + ["CommandLine"] = "app.exe" + }; + var data = new DataDictionary { { "@environment", dict } }; + + // Act + var result = data.GetValue("@environment", _serializer); + + // Assert + Assert.NotNull(result); + Assert.Equal("LEGACY-MACHINE", result.MachineName); + Assert.Equal(4, result.ProcessorCount); + Assert.Equal(8000000000L, result.TotalPhysicalMemory); + Assert.Equal("Windows", result.OSName); + Assert.Equal("10.0", result.OSVersion); + Assert.Equal("app.exe", result.CommandLine); + } + + [Fact] + public void GetValue_PascalCaseDictionaryWithErrorAndStackFrames_RecursesIntoTypedCollections() + { + // Arrange — nested typed model (StackFrame inside Error.StackTrace) with multi-word props. + var dict = new Dictionary + { + ["Message"] = "Boom", + ["Type"] = "System.Exception", + ["TargetMethod"] = new Dictionary + { + ["Name"] = "DoWork", + ["DeclaringNamespace"] = "MyApp", + ["DeclaringType"] = "Worker" + }, + ["StackTrace"] = new List + { + new Dictionary + { + ["Name"] = "DoWork", + ["DeclaringNamespace"] = "MyApp", + ["DeclaringType"] = "Worker", + ["LineNumber"] = 42 + } + } + }; + var data = new DataDictionary { { "@error", dict } }; + + // Act + var result = data.GetValue("@error", _serializer); + + // Assert + Assert.NotNull(result); + Assert.Equal("Boom", result.Message); + Assert.NotNull(result.TargetMethod); + Assert.Equal("DoWork", result.TargetMethod.Name); + Assert.Equal("MyApp", result.TargetMethod.DeclaringNamespace); + Assert.Equal("Worker", result.TargetMethod.DeclaringType); + Assert.NotNull(result.StackTrace); + Assert.Single(result.StackTrace); + Assert.Equal(42, result.StackTrace[0].LineNumber); + } + + [Fact] + public void GetValue_PascalCaseDictionaryWithError_PreservesUserDataKeysExactly() + { + // Arrange — Error.Data is a free-form Dictionary for user-provided data. + // The PascalCase keys "SomeProp" and "AnotherKey" MUST survive extraction unchanged. + var dict = new Dictionary + { + ["Message"] = "x", + ["Type"] = "T", + ["Data"] = new Dictionary + { + ["SomeProp"] = "SomeVal", + ["AnotherKey"] = "AnotherVal", + ["MixedCASE_key"] = "preserved" + } + }; + var data = new DataDictionary { { "@error", dict } }; + + // Act + var result = data.GetValue("@error", _serializer); + + // Assert + Assert.NotNull(result); + Assert.NotNull(result.Data); + Assert.Equal("SomeVal", result.Data["SomeProp"]); + Assert.Equal("AnotherVal", result.Data["AnotherKey"]); + Assert.Equal("preserved", result.Data["MixedCASE_key"]); + } + + [Fact] + public void GetValue_PascalCaseDictionaryWithRequestInfo_PreservesQueryStringAndCookieKeys() + { + // Arrange — QueryString and Cookies are Dictionary; keys are + // user-supplied (URL params, cookie names) and MUST never be transformed. + var dict = new Dictionary + { + ["HttpMethod"] = "GET", + ["QueryString"] = new Dictionary + { + ["UserId"] = "42", + ["category|root|13546"] = "outlet", + ["MixedCASE"] = "kept" + }, + ["Cookies"] = new Dictionary + { + ["SessionId"] = "abc123", + ["XSRF-TOKEN"] = "xyz" + } + }; + var data = new DataDictionary { { "@request", dict } }; + + // Act + var result = data.GetValue("@request", _serializer); + + // Assert + Assert.NotNull(result); + Assert.Equal("GET", result.HttpMethod); + Assert.NotNull(result.QueryString); + Assert.Equal("42", result.QueryString["UserId"]); + Assert.Equal("outlet", result.QueryString["category|root|13546"]); + Assert.Equal("kept", result.QueryString["MixedCASE"]); + Assert.NotNull(result.Cookies); + Assert.Equal("abc123", result.Cookies["SessionId"]); + Assert.Equal("xyz", result.Cookies["XSRF-TOKEN"]); + } + + [Fact] + public void GetValue_MixedCaseDictionaryWithRequestInfo_HandlesBothFormatsTogether() + { + // Arrange — some keys snake_case, some PascalCase (mixed legacy + new data). + var dict = new Dictionary + { + ["http_method"] = "POST", + ["ClientIpAddress"] = "10.0.0.1", + ["is_secure"] = true, + ["UserAgent"] = "Mixed/1.0" + }; + var data = new DataDictionary { { "@request", dict } }; + + // Act + var result = data.GetValue("@request", _serializer); + + // Assert + Assert.NotNull(result); + Assert.Equal("POST", result.HttpMethod); + Assert.Equal("10.0.0.1", result.ClientIpAddress); + Assert.True(result.IsSecure); + Assert.Equal("Mixed/1.0", result.UserAgent); + } } diff --git a/tests/Exceptionless.Tests/Serializer/Models/EnvironmentInfoSerializerTests.cs b/tests/Exceptionless.Tests/Serializer/Models/EnvironmentInfoSerializerTests.cs index 28605d3dce..71b4fa1441 100644 --- a/tests/Exceptionless.Tests/Serializer/Models/EnvironmentInfoSerializerTests.cs +++ b/tests/Exceptionless.Tests/Serializer/Models/EnvironmentInfoSerializerTests.cs @@ -159,4 +159,32 @@ public void SerializeToString_WithLargeMemoryValues_PreservesValues() Assert.Equal(274877906944, deserialized.TotalPhysicalMemory); Assert.Equal(137438953472, deserialized.AvailablePhysicalMemory); } + + [Fact] + public void SerializeToString_OSProperties_UseJsonPropertyNameOverride() + { + // Arrange — [JsonPropertyName] on OSName/OSVersion must produce o_s_name/o_s_version + // (not os_name/os_version from SnakeCaseLower) to match the legacy ES field names. + var env = new EnvironmentInfo + { + OSName = "Linux", + OSVersion = "6.1.0" + }; + + // Act + string? json = _serializer.SerializeToString(env); + + // Assert — verify raw JSON keys + Assert.NotNull(json); + Assert.Contains("\"o_s_name\"", json); + Assert.Contains("\"o_s_version\"", json); + Assert.DoesNotContain("\"os_name\"", json); + Assert.DoesNotContain("\"os_version\"", json); + + // Verify round-trip + var deserialized = _serializer.Deserialize(json); + Assert.NotNull(deserialized); + Assert.Equal("Linux", deserialized.OSName); + Assert.Equal("6.1.0", deserialized.OSVersion); + } } diff --git a/tests/Exceptionless.Tests/Serializer/Models/InnerErrorSerializerTests.cs b/tests/Exceptionless.Tests/Serializer/Models/InnerErrorSerializerTests.cs index c8e913fbe7..ef7099e00b 100644 --- a/tests/Exceptionless.Tests/Serializer/Models/InnerErrorSerializerTests.cs +++ b/tests/Exceptionless.Tests/Serializer/Models/InnerErrorSerializerTests.cs @@ -1,3 +1,4 @@ +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Models.Data; using Foundatio.Serializer; @@ -14,27 +15,6 @@ public InnerErrorSerializerTests(ITestOutputHelper output) : base(output) _serializer = GetService(); } - [Fact] - public void Deserialize_SnakeCaseJson_ParsesCorrectly() - { - // Arrange - /* language=json */ - const string json = """{"message":"File not found","type":"System.IO.FileNotFoundException","code":"IO_404","target_method":{"name":"ReadFile","declaring_type":"FileService","declaring_namespace":"MyApp.IO"},"stack_trace":[{"declaring_namespace":"MyApp.IO","declaring_type":"FileService","name":"ReadFile","line":42}]}"""; - - // Act - var result = _serializer.Deserialize(json); - - // Assert - Assert.NotNull(result); - Assert.Equal("File not found", result.Message); - Assert.Equal("System.IO.FileNotFoundException", result.Type); - Assert.Equal("IO_404", result.Code); - Assert.NotNull(result.TargetMethod); - Assert.Equal("ReadFile", result.TargetMethod.Name); - Assert.NotNull(result.StackTrace); - Assert.Single(result.StackTrace); - } - [Fact] public void RoundTrip_WithAllProperties_PreservesValues() { @@ -68,9 +48,6 @@ public void RoundTrip_WithAllProperties_PreservesValues() var result = _serializer.Deserialize(json); // Assert - SerializerContractAssertions.IncludesProperties(json, "stack_trace", "target_method"); - SerializerContractAssertions.ExcludesProperties(json, "StackTrace", "TargetMethod"); - Assert.NotNull(result); Assert.Equal("Object reference not set to an instance of an object.", result.Message); Assert.Equal("System.NullReferenceException", result.Type); @@ -117,4 +94,49 @@ public void RoundTrip_WithNestedInnerErrors_PreservesDepth() Assert.Equal("ARG_NULL", result.Inner.Inner.Code); Assert.Null(result.Inner.Inner.Inner); } + + [Fact] + public void Deserialize_SnakeCaseJson_ParsesCorrectly() + { + // Arrange + /* language=json */ + const string json = """{"message":"File not found","type":"System.IO.FileNotFoundException","code":"IO_404","target_method":{"name":"ReadFile","declaring_type":"FileService","declaring_namespace":"MyApp.IO"},"stack_trace":[{"declaring_namespace":"MyApp.IO","declaring_type":"FileService","name":"ReadFile","line":42}]}"""; + + // Act + var result = _serializer.Deserialize(json); + + // Assert + Assert.NotNull(result); + Assert.Equal("File not found", result.Message); + Assert.Equal("System.IO.FileNotFoundException", result.Type); + Assert.Equal("IO_404", result.Code); + Assert.NotNull(result.TargetMethod); + Assert.Equal("ReadFile", result.TargetMethod.Name); + Assert.NotNull(result.StackTrace); + Assert.Single(result.StackTrace); + } + + [Fact] + public void GetValue_InnerErrorInDictionary_DeserializesCorrectly() + { + // Arrange + var dict = new DataDictionary + { + ["@error"] = new InnerError + { + Message = "Test error", + Type = "System.Exception", + Inner = new InnerError { Message = "Cause", Type = "System.ArgumentException" } + } + }; + + // Act + var result = dict.GetValue("@error", _serializer); + + // Assert + Assert.NotNull(result); + Assert.Equal("Test error", result.Message); + Assert.NotNull(result.Inner); + Assert.Equal("Cause", result.Inner.Message); + } } diff --git a/tests/Exceptionless.Tests/Serializer/Models/LocationSerializerTests.cs b/tests/Exceptionless.Tests/Serializer/Models/LocationSerializerTests.cs index 153a35dd4a..8278609185 100644 --- a/tests/Exceptionless.Tests/Serializer/Models/LocationSerializerTests.cs +++ b/tests/Exceptionless.Tests/Serializer/Models/LocationSerializerTests.cs @@ -13,24 +13,6 @@ public LocationSerializerTests(ITestOutputHelper output) : base(output) _serializer = GetService(); } - [Fact] - public void Deserialize_SnakeCaseJson_ParsesCorrectly() - { - // Arrange - /* language=json */ - const string json = """{"country":"Canada","level1":"Ontario","level2":"York Region","locality":"Toronto"}"""; - - // Act - var result = _serializer.Deserialize(json); - - // Assert - Assert.NotNull(result); - Assert.Equal("Canada", result.Country); - Assert.Equal("Ontario", result.Level1); - Assert.Equal("York Region", result.Level2); - Assert.Equal("Toronto", result.Locality); - } - [Fact] public void RoundTrip_WithAllProperties_PreservesValues() { @@ -48,9 +30,6 @@ public void RoundTrip_WithAllProperties_PreservesValues() var result = _serializer.Deserialize(json); // Assert - SerializerContractAssertions.IncludesProperties(json, "level1", "level2"); - SerializerContractAssertions.ExcludesProperties(json, "Level1", "Level2"); - Assert.NotNull(result); Assert.Equal("United States", result.Country); Assert.Equal("Texas", result.Level1); @@ -97,4 +76,22 @@ public void RoundTrip_WithUnicodeNames_PreservesValues() Assert.Equal("東京都", result.Level1); Assert.Equal("渋谷区", result.Locality); } + + [Fact] + public void Deserialize_SnakeCaseJson_ParsesCorrectly() + { + // Arrange + /* language=json */ + const string json = """{"country":"Canada","level1":"Ontario","level2":"York Region","locality":"Toronto"}"""; + + // Act + var result = _serializer.Deserialize(json); + + // Assert + Assert.NotNull(result); + Assert.Equal("Canada", result.Country); + Assert.Equal("Ontario", result.Level1); + Assert.Equal("York Region", result.Level2); + Assert.Equal("Toronto", result.Locality); + } } diff --git a/tests/Exceptionless.Tests/Serializer/Models/ManualStackingInfoSerializerTests.cs b/tests/Exceptionless.Tests/Serializer/Models/ManualStackingInfoSerializerTests.cs index a840651f66..7801a58faa 100644 --- a/tests/Exceptionless.Tests/Serializer/Models/ManualStackingInfoSerializerTests.cs +++ b/tests/Exceptionless.Tests/Serializer/Models/ManualStackingInfoSerializerTests.cs @@ -1,3 +1,4 @@ +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Models.Data; using Foundatio.Serializer; @@ -14,6 +15,34 @@ public ManualStackingInfoSerializerTests(ITestOutputHelper output) : base(output _serializer = GetService(); } + [Fact] + public void RoundTrip_WithAllProperties_PreservesValues() + { + // Arrange + var info = new ManualStackingInfo + { + Title = "Payment Processing Error", + SignatureData = new Dictionary + { + ["payment_provider"] = "stripe", + ["error_code"] = "card_declined", + ["region"] = "US" + } + }; + + // Act + string? json = _serializer.SerializeToString(info); + var result = _serializer.Deserialize(json); + + // Assert + Assert.NotNull(result); + Assert.Equal("Payment Processing Error", result.Title); + Assert.NotNull(result.SignatureData); + Assert.Equal(3, result.SignatureData.Count); + Assert.Equal("stripe", result.SignatureData["payment_provider"]); + Assert.Equal("card_declined", result.SignatureData["error_code"]); + } + [Fact] public void Deserialize_SnakeCaseJson_ParsesCorrectly() { @@ -32,17 +61,16 @@ public void Deserialize_SnakeCaseJson_ParsesCorrectly() } [Fact] - public void RoundTrip_WithAllProperties_PreservesValues() + public void RoundTrip_WithSpecialCharacters_PreservesValues() { // Arrange var info = new ManualStackingInfo { - Title = "Payment Processing Error", + Title = "Error: \"Connection refused\" at /api/v2/events", SignatureData = new Dictionary { - ["payment_provider"] = "stripe", - ["error_code"] = "card_declined", - ["region"] = "US" + ["path"] = "/api/v2/events?filter=type:error", + ["message"] = "Connection refused: host=db.example.com, port=5432" } }; @@ -51,15 +79,9 @@ public void RoundTrip_WithAllProperties_PreservesValues() var result = _serializer.Deserialize(json); // Assert - SerializerContractAssertions.IncludesProperties(json, "signature_data"); - SerializerContractAssertions.ExcludesProperties(json, "SignatureData"); - Assert.NotNull(result); - Assert.Equal("Payment Processing Error", result.Title); - Assert.NotNull(result.SignatureData); - Assert.Equal(3, result.SignatureData.Count); - Assert.Equal("stripe", result.SignatureData["payment_provider"]); - Assert.Equal("card_declined", result.SignatureData["error_code"]); + Assert.Contains("Connection refused", result.Title); + Assert.Equal("/api/v2/events?filter=type:error", result.SignatureData!["path"]); } [Fact] @@ -78,26 +100,25 @@ public void RoundTrip_WithMinimalProperties_PreservesValues() } [Fact] - public void RoundTrip_WithSpecialCharacters_PreservesValues() + public void DataDictionary_GetValue_ManualStackingInfo_FromDictionary() { // Arrange - var info = new ManualStackingInfo + var dict = new DataDictionary { - Title = "Error: \"Connection refused\" at /api/v2/events", - SignatureData = new Dictionary + ["@stack"] = new ManualStackingInfo { - ["path"] = "/api/v2/events?filter=type:error", - ["message"] = "Connection refused: host=db.example.com, port=5432" + Title = "Custom", + SignatureData = new Dictionary { ["key"] = "val" } } }; // Act - string? json = _serializer.SerializeToString(info); - var result = _serializer.Deserialize(json); + var result = dict.GetValue("@stack", _serializer); // Assert Assert.NotNull(result); - Assert.Contains("Connection refused", result.Title); - Assert.Equal("/api/v2/events?filter=type:error", result.SignatureData!["path"]); + Assert.Equal("Custom", result.Title); + Assert.NotNull(result.SignatureData); + Assert.Equal("val", result.SignatureData["key"]); } } diff --git a/tests/Exceptionless.Tests/Serializer/Models/MethodSerializerTests.cs b/tests/Exceptionless.Tests/Serializer/Models/MethodSerializerTests.cs index 4c9803cc19..dd7bbf45d3 100644 --- a/tests/Exceptionless.Tests/Serializer/Models/MethodSerializerTests.cs +++ b/tests/Exceptionless.Tests/Serializer/Models/MethodSerializerTests.cs @@ -1,3 +1,4 @@ +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Models.Data; using Foundatio.Serializer; @@ -14,27 +15,6 @@ public MethodSerializerTests(ITestOutputHelper output) : base(output) _serializer = GetService(); } - [Fact] - public void Deserialize_SnakeCaseJson_ParsesCorrectly() - { - // Arrange - /* language=json */ - const string json = """{"is_signature_target":true,"declaring_namespace":"System","declaring_type":"String","name":"Format","module_id":1,"generic_arguments":["T"],"parameters":[{"name":"format","type":"String","type_namespace":"System"}]}"""; - - // Act - var result = _serializer.Deserialize(json); - - // Assert - Assert.NotNull(result); - Assert.True(result.IsSignatureTarget); - Assert.Equal("System", result.DeclaringNamespace); - Assert.Equal("String", result.DeclaringType); - Assert.Equal("Format", result.Name); - Assert.Equal(1, result.ModuleId); - Assert.Single(result.GenericArguments!); - Assert.Single(result.Parameters!); - } - [Fact] public void RoundTrip_WithAllProperties_PreservesValues() { @@ -59,19 +39,6 @@ public void RoundTrip_WithAllProperties_PreservesValues() var result = _serializer.Deserialize(json); // Assert - SerializerContractAssertions.IncludesProperties(json, - "is_signature_target", - "declaring_namespace", - "declaring_type", - "module_id", - "generic_arguments"); - SerializerContractAssertions.ExcludesProperties(json, - "IsSignatureTarget", - "DeclaringNamespace", - "DeclaringType", - "ModuleId", - "GenericArguments"); - Assert.NotNull(result); Assert.True(result.IsSignatureTarget); Assert.Equal("Exceptionless.Core", result.DeclaringNamespace); @@ -87,6 +54,27 @@ public void RoundTrip_WithAllProperties_PreservesValues() Assert.Equal("PersistentEvent", result.Parameters[0].Type); } + [Fact] + public void Deserialize_SnakeCaseJson_ParsesCorrectly() + { + // Arrange + /* language=json */ + const string json = """{"is_signature_target":true,"declaring_namespace":"System","declaring_type":"String","name":"Format","module_id":1,"generic_arguments":["T"],"parameters":[{"name":"format","type":"String","type_namespace":"System"}]}"""; + + // Act + var result = _serializer.Deserialize(json); + + // Assert + Assert.NotNull(result); + Assert.True(result.IsSignatureTarget); + Assert.Equal("System", result.DeclaringNamespace); + Assert.Equal("String", result.DeclaringType); + Assert.Equal("Format", result.Name); + Assert.Equal(1, result.ModuleId); + Assert.Single(result.GenericArguments!); + Assert.Single(result.Parameters!); + } + [Fact] public void RoundTrip_WithMinimalProperties_PreservesValues() { @@ -103,4 +91,27 @@ public void RoundTrip_WithMinimalProperties_PreservesValues() Assert.Null(result.DeclaringNamespace); Assert.Null(result.IsSignatureTarget); } + + [Fact] + public void GetValue_MethodInDictionary_DeserializesCorrectly() + { + // Arrange + var dict = new DataDictionary + { + ["target_method"] = new Method + { + Name = "HandleRequest", + DeclaringType = "Controller", + DeclaringNamespace = "MyApp.Web" + } + }; + + // Act + var result = dict.GetValue("target_method", _serializer); + + // Assert + Assert.NotNull(result); + Assert.Equal("HandleRequest", result.Name); + Assert.Equal("Controller", result.DeclaringType); + } } diff --git a/tests/Exceptionless.Tests/Serializer/Models/OrganizationSerializerTests.cs b/tests/Exceptionless.Tests/Serializer/Models/OrganizationSerializerTests.cs index be3c8c808c..f18ea0b28c 100644 --- a/tests/Exceptionless.Tests/Serializer/Models/OrganizationSerializerTests.cs +++ b/tests/Exceptionless.Tests/Serializer/Models/OrganizationSerializerTests.cs @@ -14,24 +14,6 @@ public OrganizationSerializerTests(ITestOutputHelper output) : base(output) _serializer = GetService(); } - [Fact] - public void Deserialize_SnakeCaseJson_ParsesCorrectly() - { - // Arrange - /* language=json */ - const string json = """{"id":"550000000000000000000004","name":"Acme Industries","plan_id":"EX_SMALL","plan_name":"Small","plan_description":"Small plan","billing_status":1,"max_events_per_month":10000,"retention_days":7,"has_premium_features":false,"max_users":5,"max_projects":10,"created_utc":"2024-01-01T00:00:00Z","updated_utc":"2024-01-01T00:00:00Z"}"""; - - // Act - var result = _serializer.Deserialize(json); - - // Assert - Assert.NotNull(result); - Assert.Equal("550000000000000000000004", result.Id); - Assert.Equal("Acme Industries", result.Name); - Assert.Equal("EX_SMALL", result.PlanId); - Assert.Equal(10000, result.MaxEventsPerMonth); - } - [Fact] public void RoundTrip_WithAllCoreProperties_PreservesValues() { @@ -61,35 +43,6 @@ public void RoundTrip_WithAllCoreProperties_PreservesValues() var result = _serializer.Deserialize(json); // Assert - SerializerContractAssertions.IncludesProperties(json, - "stripe_customer_id", - "plan_id", - "plan_name", - "plan_description", - "card_last4", - "billing_status", - "max_events_per_month", - "retention_days", - "has_premium_features", - "max_users", - "max_projects", - "created_utc", - "updated_utc"); - SerializerContractAssertions.ExcludesProperties(json, - "StripeCustomerId", - "PlanId", - "PlanName", - "PlanDescription", - "CardLast4", - "BillingStatus", - "MaxEventsPerMonth", - "RetentionDays", - "HasPremiumFeatures", - "MaxUsers", - "MaxProjects", - "CreatedUtc", - "UpdatedUtc"); - Assert.NotNull(result); Assert.Equal("550000000000000000000001", result.Id); Assert.Equal("Acme Corp", result.Name); @@ -114,8 +67,8 @@ public void RoundTrip_WithInvites_PreservesCollection() PlanId = "EX_FREE", PlanName = "Free", PlanDescription = "Free plan", - CreatedUtc = new DateTime(2024, 1, 1, 0, 0, 0, DateTimeKind.Utc), - UpdatedUtc = new DateTime(2024, 1, 2, 0, 0, 0, DateTimeKind.Utc) + CreatedUtc = DateTime.UtcNow, + UpdatedUtc = DateTime.UtcNow }; organization.Invites.Add(new Invite { @@ -151,8 +104,8 @@ public void RoundTrip_WithSuspension_PreservesValues() SuspensionNotes = "Payment failed", SuspensionDate = new DateTime(2024, 5, 1, 0, 0, 0, DateTimeKind.Utc), SuspendedByUserId = "660000000000000000000001", - CreatedUtc = new DateTime(2024, 1, 1, 0, 0, 0, DateTimeKind.Utc), - UpdatedUtc = new DateTime(2024, 1, 2, 0, 0, 0, DateTimeKind.Utc) + CreatedUtc = DateTime.UtcNow, + UpdatedUtc = DateTime.UtcNow }; // Act @@ -160,23 +113,28 @@ public void RoundTrip_WithSuspension_PreservesValues() var result = _serializer.Deserialize(json); // Assert - SerializerContractAssertions.IncludesProperties(json, - "is_suspended", - "suspension_code", - "suspension_notes", - "suspension_date", - "suspended_by_user_id"); - SerializerContractAssertions.ExcludesProperties(json, - "IsSuspended", - "SuspensionCode", - "SuspensionNotes", - "SuspensionDate", - "SuspendedByUserId"); - Assert.NotNull(result); Assert.True(result.IsSuspended); Assert.Equal(SuspensionCode.Billing, result.SuspensionCode); Assert.Equal("Payment failed", result.SuspensionNotes); Assert.Equal("660000000000000000000001", result.SuspendedByUserId); } + + [Fact] + public void Deserialize_SnakeCaseJson_ParsesCorrectly() + { + // Arrange + /* language=json */ + const string json = """{"id":"550000000000000000000004","name":"Acme Industries","plan_id":"EX_SMALL","plan_name":"Small","plan_description":"Small plan","billing_status":1,"max_events_per_month":10000,"retention_days":7,"has_premium_features":false,"max_users":5,"max_projects":10,"created_utc":"2024-01-01T00:00:00Z","updated_utc":"2024-01-01T00:00:00Z"}"""; + + // Act + var result = _serializer.Deserialize(json); + + // Assert + Assert.NotNull(result); + Assert.Equal("550000000000000000000004", result.Id); + Assert.Equal("Acme Industries", result.Name); + Assert.Equal("EX_SMALL", result.PlanId); + Assert.Equal(10000, result.MaxEventsPerMonth); + } } diff --git a/tests/Exceptionless.Tests/Serializer/Models/PersistentEventSerializerTests.cs b/tests/Exceptionless.Tests/Serializer/Models/PersistentEventSerializerTests.cs index 8d77001c8a..173fbb97d2 100644 --- a/tests/Exceptionless.Tests/Serializer/Models/PersistentEventSerializerTests.cs +++ b/tests/Exceptionless.Tests/Serializer/Models/PersistentEventSerializerTests.cs @@ -1,6 +1,6 @@ -using System.Text.Json; using Exceptionless.Core.Models; using Exceptionless.Core.Models.Data; +using Exceptionless.Core.Serialization; using Foundatio.Serializer; using Xunit; @@ -14,13 +14,11 @@ namespace Exceptionless.Tests.Serializer.Models; public class PersistentEventSerializerTests : TestWithServices { private readonly ITextSerializer _serializer; - private readonly JsonSerializerOptions _jsonOptions; private static readonly DateTimeOffset FixedDate = new(2024, 1, 15, 12, 0, 0, TimeSpan.Zero); public PersistentEventSerializerTests(ITestOutputHelper output) : base(output) { _serializer = GetService(); - _jsonOptions = GetService(); TimeProvider.SetUtcNow(FixedDate); } @@ -36,6 +34,7 @@ public void SerializeToString_CompleteEvent_PreservesAllProperties() StackId = "stack012", Type = Event.KnownTypes.Error, Message = "Test error occurred", + CreatedUtc = new DateTime(2024, 1, 15, 12, 0, 0, DateTimeKind.Utc), Date = FixedDate, Tags = ["production", "critical"], Geo = "37.7749,-122.4194", @@ -60,6 +59,51 @@ public void SerializeToString_CompleteEvent_PreservesAllProperties() Assert.Equal(3, deserialized.Count); } + [Fact] + public void SerializeToString_CreatedUtc_PreservesUtcStorageFormat() + { + // Arrange + var ev = new PersistentEvent + { + Id = "event123", + OrganizationId = "org456", + ProjectId = "proj789", + StackId = "stack012", + Type = Event.KnownTypes.Error, + CreatedUtc = new DateTime(2024, 1, 15, 12, 0, 0, DateTimeKind.Utc), + Date = FixedDate + }; + + // Act + string? json = _serializer.SerializeToString(ev); + + // Assert + Assert.Contains("\"created_utc\":\"2024-01-15T12:00:00Z\"", json); + } + + [Fact] + public void ConfigureExceptionlessApiDefaults_CreatedUtc_UsesStandardUtcFormat() + { + // Arrange + var options = new System.Text.Json.JsonSerializerOptions().ConfigureExceptionlessApiDefaults(); + var ev = new PersistentEvent + { + Id = "event123", + OrganizationId = "org456", + ProjectId = "proj789", + StackId = "stack012", + Type = Event.KnownTypes.Error, + CreatedUtc = new DateTime(2024, 1, 15, 12, 0, 0, DateTimeKind.Utc), + Date = FixedDate + }; + + // Act + string json = System.Text.Json.JsonSerializer.Serialize(ev, options); + + // Assert + Assert.Contains("\"created_utc\":\"2024-01-15T12:00:00Z\"", json); + } + [Fact] public void SerializeToString_MinimalEvent_PreservesProperties() { @@ -105,7 +149,7 @@ public void Deserialize_EventWithUserInfo_PreservesTypedUserInfo() // Assert Assert.NotNull(deserialized); - var userInfo = deserialized.GetUserIdentity(_jsonOptions); + var userInfo = deserialized.GetUserIdentity(_serializer, _logger); Assert.NotNull(userInfo); Assert.Equal("user@example.com", userInfo.Identity); Assert.Equal("Test User", userInfo.Name); @@ -146,7 +190,7 @@ public void Deserialize_EventWithError_PreservesTypedError() // Assert Assert.NotNull(deserialized); - var error = deserialized.GetError(_jsonOptions); + var error = deserialized.GetError(_serializer, _logger); Assert.NotNull(error); Assert.Equal("Test exception", error.Message); Assert.Equal("System.InvalidOperationException", error.Type); @@ -183,7 +227,7 @@ public void Deserialize_EventWithRequestInfo_PreservesTypedRequestInfo() // Assert Assert.NotNull(deserialized); - var request = deserialized.GetRequestInfo(_jsonOptions); + var request = deserialized.GetRequestInfo(_serializer, _logger); Assert.NotNull(request); Assert.Equal("POST", request.HttpMethod); Assert.Equal("/api/events", request.Path); @@ -215,7 +259,7 @@ public void Deserialize_EventWithEnvironmentInfo_PreservesTypedEnvironmentInfo() // Assert Assert.NotNull(deserialized); - var env = deserialized.GetEnvironmentInfo(_jsonOptions); + var env = deserialized.GetEnvironmentInfo(_serializer, _logger); Assert.NotNull(env); Assert.Equal("PROD-SERVER-01", env.MachineName); Assert.Equal(8, env.ProcessorCount); @@ -270,9 +314,9 @@ public void Deserialize_EventWithAllKnownDataKeys_PreservesAllTypes() // Assert Assert.NotNull(deserialized); - Assert.NotNull(deserialized.GetUserIdentity(_jsonOptions)); - Assert.NotNull(deserialized.GetRequestInfo(_jsonOptions)); - Assert.NotNull(deserialized.GetEnvironmentInfo(_jsonOptions)); + Assert.NotNull(deserialized.GetUserIdentity(_serializer, _logger)); + Assert.NotNull(deserialized.GetRequestInfo(_serializer, _logger)); + Assert.NotNull(deserialized.GetEnvironmentInfo(_serializer, _logger)); Assert.Equal("1.0.0", deserialized.GetVersion()); Assert.Equal("Error", deserialized.GetLevel()); } @@ -328,7 +372,7 @@ public void Deserialize_JsonWithTypedUserData_RetrievesTypedUserInfo() // Assert Assert.NotNull(ev); - var userInfo = ev.GetUserIdentity(_jsonOptions); + var userInfo = ev.GetUserIdentity(_serializer, _logger); Assert.NotNull(userInfo); Assert.Equal("parsed@example.com", userInfo.Identity); Assert.Equal("Parsed User", userInfo.Name); @@ -352,20 +396,28 @@ public void Deserialize_JsonWithNestedDataProperties_PreservesNestedStructure() } [Fact] - public void Deserialize_JsonWithUnknownRootProperties_IgnoresUnknownProperties() + public void Deserialize_JsonWithUnknownRootProperties_MergesIntoData() { - // Arrange + // Arrange: unknown root properties are captured via [JsonExtensionData] and + // merged into Data dictionary via Event.OnDeserialized(). /* language=json */ - const string json = """{"id":"unk-1","organization_id":"org1","project_id":"proj1","type":"log","message":"Test","date":"2024-01-15T12:00:00+00:00","unknown_property":"should_be_ignored","another_unknown":123,"tags":[],"count":1,"data":{},"is_first_occurrence":false,"is_fixed":false,"is_hidden":false}"""; + const string json = """{"id":"unk-1","organization_id":"org1","project_id":"proj1","type":"log","message":"Test","date":"2024-01-15T12:00:00+00:00","unknown_property":"should_be_captured","another_unknown":123,"tags":[],"count":1,"data":{},"is_first_occurrence":false,"is_fixed":false,"is_hidden":false}"""; // Act var ev = _serializer.Deserialize(json); - // Assert + // Assert — known properties are parsed correctly Assert.NotNull(ev); Assert.Equal("unk-1", ev.Id); Assert.Equal("log", ev.Type); Assert.Equal("Test", ev.Message); + + // Unknown root properties are merged into Data via OnDeserialized + Assert.NotNull(ev.Data); + Assert.True(ev.Data.TryGetValue("unknown_property", out var unknownProperty)); + Assert.Equal("should_be_captured", unknownProperty); + Assert.True(ev.Data.TryGetValue("another_unknown", out var anotherUnknown)); + Assert.Equal(123, anotherUnknown); } [Fact] diff --git a/tests/Exceptionless.Tests/Serializer/Models/SavedViewSerializerTests.cs b/tests/Exceptionless.Tests/Serializer/Models/SavedViewSerializerTests.cs index 783125311c..685f337ed9 100644 --- a/tests/Exceptionless.Tests/Serializer/Models/SavedViewSerializerTests.cs +++ b/tests/Exceptionless.Tests/Serializer/Models/SavedViewSerializerTests.cs @@ -13,24 +13,6 @@ public SavedViewSerializerTests(ITestOutputHelper output) : base(output) _serializer = GetService(); } - [Fact] - public void Deserialize_SnakeCaseJson_ParsesCorrectly() - { - // Arrange - /* language=json */ - const string json = """{"id":"770000000000000000000003","organization_id":"550000000000000000000001","created_by_user_id":"660000000000000000000001","is_default":false,"name":"Error Stream","view_type":"stream","version":1,"created_utc":"2024-02-20T14:30:00Z","updated_utc":"2024-02-20T14:30:00Z"}"""; - - // Act - var result = _serializer.Deserialize(json); - - // Assert - Assert.NotNull(result); - Assert.Equal("770000000000000000000003", result.Id); - Assert.Equal("Error Stream", result.Name); - Assert.Equal("stream", result.ViewType); - Assert.Equal("660000000000000000000001", result.CreatedByUserId); - } - [Fact] public void RoundTrip_WithAllProperties_PreservesValues() { @@ -63,25 +45,6 @@ public void RoundTrip_WithAllProperties_PreservesValues() var result = _serializer.Deserialize(json); // Assert - SerializerContractAssertions.IncludesProperties(json, - "organization_id", - "user_id", - "created_by_user_id", - "updated_by_user_id", - "filter_definitions", - "view_type", - "created_utc", - "updated_utc"); - SerializerContractAssertions.ExcludesProperties(json, - "OrganizationId", - "UserId", - "CreatedByUserId", - "UpdatedByUserId", - "FilterDefinitions", - "ViewType", - "CreatedUtc", - "UpdatedUtc"); - Assert.NotNull(result); Assert.Equal("770000000000000000000001", result.Id); Assert.Equal("550000000000000000000001", result.OrganizationId); @@ -109,8 +72,8 @@ public void RoundTrip_WithMinimalProperties_PreservesValues() CreatedByUserId = "660000000000000000000001", Name = "All Events", ViewType = "events", - CreatedUtc = new DateTime(2024, 1, 1, 0, 0, 0, DateTimeKind.Utc), - UpdatedUtc = new DateTime(2024, 1, 2, 0, 0, 0, DateTimeKind.Utc) + CreatedUtc = DateTime.UtcNow, + UpdatedUtc = DateTime.UtcNow }; // Act @@ -125,4 +88,22 @@ public void RoundTrip_WithMinimalProperties_PreservesValues() Assert.Null(result.Filter); Assert.Null(result.Time); } + + [Fact] + public void Deserialize_SnakeCaseJson_ParsesCorrectly() + { + // Arrange + /* language=json */ + const string json = """{"id":"770000000000000000000003","organization_id":"550000000000000000000001","created_by_user_id":"660000000000000000000001","is_default":false,"name":"Error Stream","view_type":"stream","version":1,"created_utc":"2024-02-20T14:30:00Z","updated_utc":"2024-02-20T14:30:00Z"}"""; + + // Act + var result = _serializer.Deserialize(json); + + // Assert + Assert.NotNull(result); + Assert.Equal("770000000000000000000003", result.Id); + Assert.Equal("Error Stream", result.Name); + Assert.Equal("stream", result.ViewType); + Assert.Equal("660000000000000000000001", result.CreatedByUserId); + } } diff --git a/tests/Exceptionless.Tests/Serializer/Models/SubmissionClientSerializerTests.cs b/tests/Exceptionless.Tests/Serializer/Models/SubmissionClientSerializerTests.cs index c3eb3c9a38..b7d862da51 100644 --- a/tests/Exceptionless.Tests/Serializer/Models/SubmissionClientSerializerTests.cs +++ b/tests/Exceptionless.Tests/Serializer/Models/SubmissionClientSerializerTests.cs @@ -13,23 +13,6 @@ public SubmissionClientSerializerTests(ITestOutputHelper output) : base(output) _serializer = GetService(); } - [Fact] - public void Deserialize_SnakeCaseJson_ParsesCorrectly() - { - // Arrange - /* language=json */ - const string json = """{"ip_address":"10.0.0.1","user_agent":"Mozilla/5.0","version":"1.0.0"}"""; - - // Act - var result = _serializer.Deserialize(json); - - // Assert - Assert.NotNull(result); - Assert.Equal("10.0.0.1", result.IpAddress); - Assert.Equal("Mozilla/5.0", result.UserAgent); - Assert.Equal("1.0.0", result.Version); - } - [Fact] public void RoundTrip_WithAllProperties_PreservesValues() { @@ -46,9 +29,6 @@ public void RoundTrip_WithAllProperties_PreservesValues() var result = _serializer.Deserialize(json); // Assert - SerializerContractAssertions.IncludesProperties(json, "ip_address", "user_agent"); - SerializerContractAssertions.ExcludesProperties(json, "IpAddress", "UserAgent"); - Assert.NotNull(result); Assert.Equal("192.168.1.100", result.IpAddress); Assert.Equal("exceptionless/1.0.0", result.UserAgent); @@ -75,6 +55,23 @@ public void RoundTrip_WithIPv6Address_PreservesValues() Assert.Equal("::ffff:192.168.1.1", result.IpAddress); } + [Fact] + public void Deserialize_SnakeCaseJson_ParsesCorrectly() + { + // Arrange + /* language=json */ + const string json = """{"ip_address":"10.0.0.1","user_agent":"Mozilla/5.0","version":"1.0.0"}"""; + + // Act + var result = _serializer.Deserialize(json); + + // Assert + Assert.NotNull(result); + Assert.Equal("10.0.0.1", result.IpAddress); + Assert.Equal("Mozilla/5.0", result.UserAgent); + Assert.Equal("1.0.0", result.Version); + } + [Fact] public void RoundTrip_WithMinimalProperties_PreservesValues() { diff --git a/tests/Exceptionless.Tests/Serializer/Models/TokenSerializerTests.cs b/tests/Exceptionless.Tests/Serializer/Models/TokenSerializerTests.cs index 16e689ab92..4855f5da04 100644 --- a/tests/Exceptionless.Tests/Serializer/Models/TokenSerializerTests.cs +++ b/tests/Exceptionless.Tests/Serializer/Models/TokenSerializerTests.cs @@ -13,26 +13,6 @@ public TokenSerializerTests(ITestOutputHelper output) : base(output) _serializer = GetService(); } - [Fact] - public void Deserialize_SnakeCaseJson_ParsesCorrectly() - { - // Arrange - /* language=json */ - const string json = """{"id":"650000000000000000000004","organization_id":"550000000000000000000001","project_id":"540000000000000000000001","type":1,"scopes":["client","user"],"notes":"Test token","is_disabled":false,"created_by":"user1","created_utc":"2024-03-15T10:00:00Z","updated_utc":"2024-03-15T10:00:00Z"}"""; - - // Act - var result = _serializer.Deserialize(json); - - // Assert - Assert.NotNull(result); - Assert.Equal("650000000000000000000004", result.Id); - Assert.Equal("550000000000000000000001", result.OrganizationId); - Assert.Equal(TokenType.Access, result.Type); - Assert.Equal(2, result.Scopes.Count); - Assert.Contains("client", result.Scopes); - Assert.Contains("user", result.Scopes); - } - [Fact] public void RoundTrip_WithAccessToken_PreservesValues() { @@ -55,19 +35,6 @@ public void RoundTrip_WithAccessToken_PreservesValues() var result = _serializer.Deserialize(json); // Assert - SerializerContractAssertions.IncludesProperties(json, - "organization_id", - "project_id", - "created_by", - "created_utc", - "updated_utc"); - SerializerContractAssertions.ExcludesProperties(json, - "OrganizationId", - "ProjectId", - "CreatedBy", - "CreatedUtc", - "UpdatedUtc"); - Assert.NotNull(result); Assert.Equal("650000000000000000000001", result.Id); Assert.Equal("550000000000000000000001", result.OrganizationId); @@ -77,6 +44,38 @@ public void RoundTrip_WithAccessToken_PreservesValues() Assert.Equal("Production API key", result.Notes); } + [Fact] + public void RoundTrip_WithUserScopedToken_PreservesValues() + { + // Arrange + var token = new Token + { + Id = "650000000000000000000002", + OrganizationId = "", + ProjectId = "", + UserId = "660000000000000000000001", + DefaultProjectId = "540000000000000000000001", + Type = TokenType.Authentication, + Refresh = "refresh_token_abc", + ExpiresUtc = new DateTime(2025, 1, 1, 0, 0, 0, DateTimeKind.Utc), + CreatedBy = "system", + CreatedUtc = DateTime.UtcNow, + UpdatedUtc = DateTime.UtcNow + }; + + // Act + string? json = _serializer.SerializeToString(token); + var result = _serializer.Deserialize(json); + + // Assert + Assert.NotNull(result); + Assert.Equal("660000000000000000000001", result.UserId); + Assert.Equal("540000000000000000000001", result.DefaultProjectId); + Assert.Equal(TokenType.Authentication, result.Type); + Assert.Equal("refresh_token_abc", result.Refresh); + Assert.NotNull(result.ExpiresUtc); + } + [Fact] public void RoundTrip_WithDisabledSuspended_PreservesFlags() { @@ -90,8 +89,8 @@ public void RoundTrip_WithDisabledSuspended_PreservesFlags() IsDisabled = true, IsSuspended = true, CreatedBy = "admin", - CreatedUtc = new DateTime(2024, 1, 1, 0, 0, 0, DateTimeKind.Utc), - UpdatedUtc = new DateTime(2024, 1, 2, 0, 0, 0, DateTimeKind.Utc) + CreatedUtc = DateTime.UtcNow, + UpdatedUtc = DateTime.UtcNow }; // Act @@ -99,46 +98,28 @@ public void RoundTrip_WithDisabledSuspended_PreservesFlags() var result = _serializer.Deserialize(json); // Assert - SerializerContractAssertions.IncludesProperties(json, "is_disabled", "is_suspended"); - SerializerContractAssertions.ExcludesProperties(json, "IsDisabled", "IsSuspended"); - Assert.NotNull(result); Assert.True(result.IsDisabled); Assert.True(result.IsSuspended); } [Fact] - public void RoundTrip_WithUserScopedToken_PreservesValues() + public void Deserialize_SnakeCaseJson_ParsesCorrectly() { // Arrange - var token = new Token - { - Id = "650000000000000000000002", - OrganizationId = "", - ProjectId = "", - UserId = "660000000000000000000001", - DefaultProjectId = "540000000000000000000001", - Type = TokenType.Authentication, - Refresh = "refresh_token_abc", - ExpiresUtc = new DateTime(2025, 1, 1, 0, 0, 0, DateTimeKind.Utc), - CreatedBy = "system", - CreatedUtc = new DateTime(2024, 1, 1, 0, 0, 0, DateTimeKind.Utc), - UpdatedUtc = new DateTime(2024, 1, 2, 0, 0, 0, DateTimeKind.Utc) - }; + /* language=json */ + const string json = """{"id":"650000000000000000000004","organization_id":"550000000000000000000001","project_id":"540000000000000000000001","type":1,"scopes":["client","user"],"notes":"Test token","is_disabled":false,"created_by":"user1","created_utc":"2024-03-15T10:00:00Z","updated_utc":"2024-03-15T10:00:00Z"}"""; // Act - string? json = _serializer.SerializeToString(token); var result = _serializer.Deserialize(json); // Assert - SerializerContractAssertions.IncludesProperties(json, "user_id", "default_project_id", "expires_utc"); - SerializerContractAssertions.ExcludesProperties(json, "UserId", "DefaultProjectId", "ExpiresUtc"); - Assert.NotNull(result); - Assert.Equal("660000000000000000000001", result.UserId); - Assert.Equal("540000000000000000000001", result.DefaultProjectId); - Assert.Equal(TokenType.Authentication, result.Type); - Assert.Equal("refresh_token_abc", result.Refresh); - Assert.NotNull(result.ExpiresUtc); + Assert.Equal("650000000000000000000004", result.Id); + Assert.Equal("550000000000000000000001", result.OrganizationId); + Assert.Equal(TokenType.Access, result.Type); + Assert.Equal(2, result.Scopes.Count); + Assert.Contains("client", result.Scopes); + Assert.Contains("user", result.Scopes); } } diff --git a/tests/Exceptionless.Tests/Serializer/Models/UserDescriptionSerializerTests.cs b/tests/Exceptionless.Tests/Serializer/Models/UserDescriptionSerializerTests.cs index 7e73fd17b1..53b841cdfd 100644 --- a/tests/Exceptionless.Tests/Serializer/Models/UserDescriptionSerializerTests.cs +++ b/tests/Exceptionless.Tests/Serializer/Models/UserDescriptionSerializerTests.cs @@ -1,3 +1,4 @@ +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Models.Data; using Foundatio.Serializer; @@ -14,23 +15,6 @@ public UserDescriptionSerializerTests(ITestOutputHelper output) : base(output) _serializer = GetService(); } - [Fact] - public void Deserialize_SnakeCaseJson_ParsesCorrectly() - { - // Arrange - /* language=json */ - const string json = """{"email_address":"test+tags@example.org","description":"Steps: 1. Open page 2. Click button 3. See error","data":{"screenshot":"base64data"}}"""; - - // Act - var result = _serializer.Deserialize(json); - - // Assert - Assert.NotNull(result); - Assert.Equal("test+tags@example.org", result.EmailAddress); - Assert.Contains("Steps:", result.Description); - Assert.NotNull(result.Data); - } - [Fact] public void RoundTrip_WithAllProperties_PreservesValues() { @@ -51,9 +35,6 @@ public void RoundTrip_WithAllProperties_PreservesValues() var result = _serializer.Deserialize(json); // Assert - SerializerContractAssertions.IncludesProperties(json, "email_address"); - SerializerContractAssertions.ExcludesProperties(json, "EmailAddress"); - Assert.NotNull(result); Assert.Equal("user@example.com", result.EmailAddress); Assert.Equal("The app crashed when I clicked the submit button.", result.Description); @@ -61,6 +42,23 @@ public void RoundTrip_WithAllProperties_PreservesValues() Assert.Equal("Chrome 120", result.Data["browser"]); } + [Fact] + public void Deserialize_SnakeCaseJson_ParsesCorrectly() + { + // Arrange + /* language=json */ + const string json = """{"email_address":"test+tags@example.org","description":"Steps: 1. Open page 2. Click button 3. See error","data":{"screenshot":"base64data"}}"""; + + // Act + var result = _serializer.Deserialize(json); + + // Assert + Assert.NotNull(result); + Assert.Equal("test+tags@example.org", result.EmailAddress); + Assert.Contains("Steps:", result.Description); + Assert.NotNull(result.Data); + } + [Fact] public void RoundTrip_WithMinimalProperties_PreservesValues() { @@ -76,4 +74,26 @@ public void RoundTrip_WithMinimalProperties_PreservesValues() Assert.Equal("It broke", result.Description); Assert.Null(result.EmailAddress); } + + [Fact] + public void DataDictionary_GetValue_UserDescription_FromDictionary() + { + // Arrange + var dict = new DataDictionary + { + ["@user_description"] = new UserDescription + { + EmailAddress = "feedback@test.com", + Description = "Needs improvement" + } + }; + + // Act + var result = dict.GetValue("@user_description", _serializer); + + // Assert + Assert.NotNull(result); + Assert.Equal("feedback@test.com", result.EmailAddress); + Assert.Equal("Needs improvement", result.Description); + } } diff --git a/tests/Exceptionless.Tests/Serializer/ObjectToInferredTypesConverterTests.cs b/tests/Exceptionless.Tests/Serializer/ObjectToInferredTypesConverterTests.cs index cfcba80357..b3aa7e8714 100644 --- a/tests/Exceptionless.Tests/Serializer/ObjectToInferredTypesConverterTests.cs +++ b/tests/Exceptionless.Tests/Serializer/ObjectToInferredTypesConverterTests.cs @@ -1,3 +1,5 @@ +using System.Text.Json; +using Exceptionless.Core.Serialization; using Foundatio.Serializer; using Xunit; @@ -130,6 +132,48 @@ public void Read_ScientificNotation_ReturnsDouble() Assert.Equal(12300000000m, (decimal)result["value"]!); } + [Fact] + public void Read_ScientificNotationOutsideDecimalRange_ReturnsDouble() + { + // Arrange + /* language=json */ + const string json = """{"value": 1e100}"""; + + // Act + var result = _serializer.Deserialize>(json); + + // Assert + Assert.NotNull(result); + Assert.IsType(result["value"]); + Assert.Equal(1e100d, (double)result["value"]!); + } + + [Fact] + public void Read_SubnormalDouble_ReturnsDouble() + { + /* language=json */ + const string json = """{"value": 5e-324}"""; + + var result = _serializer.Deserialize>(json); + + Assert.NotNull(result); + Assert.IsType(result["value"]); + Assert.Equal(5e-324d, (double)result["value"]!); + } + + [Fact] + public void Read_TinyNegativeDouble_ReturnsDouble() + { + /* language=json */ + const string json = """{"value": -1.23456789e-100}"""; + + var result = _serializer.Deserialize>(json); + + Assert.NotNull(result); + Assert.IsType(result["value"]); + Assert.Equal(-1.23456789e-100d, (double)result["value"]!); + } + [Fact] public void Read_PlainString_ReturnsString() { @@ -218,9 +262,10 @@ public void Read_Iso8601WithOffset_ReturnsDateTimeOffsetWithOffset() } [Fact] - public void Read_DateOnly_ReturnsDateTimeOffset() + public void Read_DateOnly_ReturnsString() { - // Arrange + // Arrange - date-only strings without time component are preserved as strings + // to match legacy Newtonsoft behavior (DateParseHandling.None for Data dictionary) /* language=json */ const string json = """{"date": "2024-01-15"}"""; @@ -229,7 +274,8 @@ public void Read_DateOnly_ReturnsDateTimeOffset() // Assert Assert.NotNull(result); - Assert.IsType(result["date"]); + Assert.IsType(result["date"]); + Assert.Equal("2024-01-15", result["date"]); } [Fact] @@ -645,8 +691,83 @@ public void Read_NumberAtInt64Boundary_HandlesCorrectly() Assert.IsType(result1["value"]); Assert.Equal(Int64.MaxValue, result1["value"]); - // Assert - Number exceeding long.MaxValue becomes double + // Assert - Number exceeding long.MaxValue but fitting decimal remains decimal Assert.NotNull(result2); Assert.IsType(result2["value"]); } + + [Fact] + public void Read_IntegerOutsideDecimalRange_ReturnsDouble() + { + // Arrange + /* language=json */ + const string json = """{"value": 999999999999999999999999999999999999999999999999999999999999}"""; + + // Act + var result = _serializer.Deserialize>(json); + + // Assert + Assert.NotNull(result); + Assert.IsType(result["value"]); + Assert.Equal(1e60d, (double)result["value"]!, 12); + } + + [Fact] + public void ConvertJsonElement_LargeExponent_ReturnsDouble() + { + // Arrange: 1e100 exceeds decimal range (~7.9×10²⁸), must not throw OverflowException + using var doc = JsonDocument.Parse("1e100"); + var element = doc.RootElement; + + // Act + var result = JsonElementConverter.Convert(element); + + // Assert + Assert.IsType(result); + Assert.Equal(1e100d, (double)result); + } + + [Fact] + public void ConvertJsonElement_SmallExponent_ReturnsDecimal() + { + // Arrange: 1.23e10 fits in decimal + using var doc = JsonDocument.Parse("1.23e10"); + var element = doc.RootElement; + + // Act + var result = JsonElementConverter.Convert(element); + + // Assert + Assert.IsType(result); + Assert.Equal(12300000000m, (decimal)result); + } + + [Fact] + public void ConvertJsonElement_VeryLargeInteger_ReturnsDecimal() + { + // Arrange: 9223372036854775808 (long.MaxValue + 1) doesn't fit in long + using var doc = JsonDocument.Parse("9223372036854775808"); + var element = doc.RootElement; + + // Act + var result = JsonElementConverter.Convert(element); + + // Assert + Assert.IsType(result); + } + + [Fact] + public void ConvertJsonElement_IntegerOutsideDecimalRange_ReturnsDouble() + { + // Arrange: integer exceeds decimal range, must not throw OverflowException + using var doc = JsonDocument.Parse("999999999999999999999999999999999999999999999999999999999999"); + var element = doc.RootElement; + + // Act + var result = JsonElementConverter.Convert(element); + + // Assert + Assert.IsType(result); + Assert.Equal(1e60d, (double)result, 12); + } } diff --git a/tests/Exceptionless.Tests/Serializer/SerializerTests.cs b/tests/Exceptionless.Tests/Serializer/SerializerTests.cs index ee344de170..f8cdac9f05 100644 --- a/tests/Exceptionless.Tests/Serializer/SerializerTests.cs +++ b/tests/Exceptionless.Tests/Serializer/SerializerTests.cs @@ -2,11 +2,8 @@ using Exceptionless.Core.Models; using Exceptionless.Core.Models.Data; using Exceptionless.Core.Services; -using Exceptionless.Serializer; -using Foundatio.Repositories.Extensions; +using Exceptionless.Tests.Utility; using Foundatio.Serializer; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; using Xunit; namespace Exceptionless.Tests.Serializer; @@ -21,36 +18,66 @@ public SerializerTests(ITestOutputHelper output) : base(output) } [Fact] - public void CanDeserializeEventWithUnknownNamesAndProperties() + public void SerializeToString_HtmlSensitiveCharacters_EscapesDangerousCharacters() { - const string json = @"{""tags"":[""One"",""Two""],""reference_id"":""12"",""Message"":""Hello"",""SomeString"":""Hi"",""SomeBool"":false,""SomeNum"":1,""UnknownProp"":{""Blah"":""SomeVal""},""Some"":{""Blah"":""SomeVal""},""@error"":{""Message"":""SomeVal"",""SomeProp"":""SomeVal""},""Some2"":""{\""Blah\"":\""SomeVal\""}"",""UnknownSerializedProp"":""{\""Blah\"":\""SomeVal\""}""}"; - var settings = new JsonSerializerSettings(); - var knownDataTypes = new Dictionary - { - { "Some", typeof(SomeModel) }, - { "Some2", typeof(SomeModel) }, - { Event.KnownDataKeys.Error, typeof(Error) } - }; - settings.Converters.Add(new DataObjectConverter(_logger, knownDataTypes)); - settings.Converters.Add(new DataObjectConverter(_logger)); - - var ev = json.FromJson(settings); - Assert.NotNull(ev?.Data); - - Assert.Equal(8, ev.Data.Count); - Assert.Equal("Hi", ev.Data.GetString("SomeString")); - Assert.False(ev.Data!.GetBoolean("SomeBool")); - Assert.Equal(1L, ev.Data["SomeNum"]); - Assert.Equal(typeof(JObject), ev.Data["UnknownProp"]?.GetType()); - Assert.Equal(typeof(JObject), ev.Data["UnknownSerializedProp"]?.GetType()); - Assert.Equal("SomeVal", (string)((dynamic)ev.Data["UnknownProp"]!)?.Blah!); - Assert.Equal(typeof(SomeModel), ev.Data["Some"]?.GetType()); - Assert.Equal(typeof(SomeModel), ev.Data["Some2"]?.GetType()); - Assert.Equal("SomeVal", (ev.Data["Some"] as SomeModel)?.Blah); - Assert.Equal(typeof(Error), ev.Data[Event.KnownDataKeys.Error]?.GetType()); - Assert.Equal("SomeVal", ((Error)ev.Data[Event.KnownDataKeys.Error]!)?.Message); - Assert.Single(((Error)ev.Data[Event.KnownDataKeys.Error]!)?.Data!); - Assert.Equal("SomeVal", ((Error)ev.Data[Event.KnownDataKeys.Error]!)?.Data?["SomeProp"]); + // Arrange + var payload = new { Text = "