diff --git a/src/Exceptionless.Core/Bootstrapper.cs b/src/Exceptionless.Core/Bootstrapper.cs index ee1db07f1e..5bd772047f 100644 --- a/src/Exceptionless.Core/Bootstrapper.cs +++ b/src/Exceptionless.Core/Bootstrapper.cs @@ -107,6 +107,7 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO handlers.Register(s.GetRequiredService); handlers.Register(s.GetRequiredService); handlers.Register(s.GetRequiredService); + handlers.Register(s.GetRequiredService); handlers.Register(s.GetRequiredService); handlers.Register(s.GetRequiredService); handlers.Register(s.GetRequiredService); diff --git a/src/Exceptionless.Core/Jobs/CleanupDataJob.cs b/src/Exceptionless.Core/Jobs/CleanupDataJob.cs index f456e89f56..8929aabe9e 100644 --- a/src/Exceptionless.Core/Jobs/CleanupDataJob.cs +++ b/src/Exceptionless.Core/Jobs/CleanupDataJob.cs @@ -2,6 +2,7 @@ using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Repositories; +using Exceptionless.Core.Repositories.Queries; using Exceptionless.Core.Services; using Exceptionless.DateTimeExtensions; using Foundatio.Caching; @@ -220,7 +221,7 @@ private async Task RemoveStacksAsync(IReadOnlyCollection stacks, JobConte long removedEvents = await _eventRepository.RemoveAllByStackIdsAsync(stackIds); await _stackRepository.RemoveAsync(stacks); foreach (var orgGroup in stacks.GroupBy(s => (s.OrganizationId, s.ProjectId))) - await _cacheClient.RemoveByPrefixAsync(String.Concat("stack-filter:", orgGroup.Key.OrganizationId, ":", orgGroup.Key.ProjectId)); + await _cacheClient.RemoveByPrefixAsync(EventStackFilterQueryBuilder.GetScopedCachePrefix(orgGroup.Key.OrganizationId, orgGroup.Key.ProjectId)); _logger.RemoveStacksComplete(stackIds.Length, removedEvents); } diff --git a/src/Exceptionless.Core/Jobs/WorkItemHandlers/GenerateSampleEventsWorkItemHandler.cs b/src/Exceptionless.Core/Jobs/WorkItemHandlers/GenerateSampleEventsWorkItemHandler.cs index cb89024b5f..a02cc441e0 100644 --- a/src/Exceptionless.Core/Jobs/WorkItemHandlers/GenerateSampleEventsWorkItemHandler.cs +++ b/src/Exceptionless.Core/Jobs/WorkItemHandlers/GenerateSampleEventsWorkItemHandler.cs @@ -35,7 +35,12 @@ public GenerateSampleEventsWorkItemHandler( public override Task GetWorkItemLockAsync(object workItem, CancellationToken cancellationToken = default) { - return _lockProvider.TryAcquireAsync(nameof(GenerateSampleEventsWorkItemHandler), TimeSpan.FromMinutes(30), cancellationToken); + var generateSampleEventsWorkItem = (GenerateSampleEventsWorkItem)workItem; + string cacheKey = IsProjectScoped(generateSampleEventsWorkItem) + ? $"{nameof(GenerateSampleEventsWorkItemHandler)}:{generateSampleEventsWorkItem.ProjectId}" + : nameof(GenerateSampleEventsWorkItemHandler); + + return _lockProvider.TryAcquireAsync(cacheKey, TimeSpan.FromMinutes(30), cancellationToken); } public override async Task HandleItemAsync(WorkItemContext context) @@ -43,13 +48,20 @@ public override async Task HandleItemAsync(WorkItemContext context) var workItem = context.GetData()!; int eventCount = Math.Clamp(workItem.EventCount, 1, 10000); int daysBack = Math.Clamp(workItem.DaysBack, 1, 365); + int acceptedDaysBack = Math.Min(daysBack, 3); - Log.LogInformation("Generating {EventCount} sample events over {DaysBack} days", eventCount, daysBack); - await context.ReportProgressAsync(0, $"Generating {eventCount} sample events"); + Log.LogInformation("Generating {EventCount} sample events over {DaysBack} days", eventCount, acceptedDaysBack); + await context.ReportProgressAsync(0, $"Generating {eventCount} sample events over {acceptedDaysBack} days"); var generator = new RandomEventGenerator(_timeProvider); var utcNow = _timeProvider.GetUtcNow().UtcDateTime; - var minDate = utcNow.AddDays(-daysBack); + var minDate = utcNow.AddDays(-acceptedDaysBack); + + if (IsProjectScoped(workItem)) + { + await GenerateProjectSampleEventsAsync(context, generator, workItem, eventCount, minDate, utcNow); + return; + } var projectResults = await _projectRepository.GetByOrganizationIdAsync(SampleDataService.TEST_ORG_ID); var projectList = projectResults.Documents.ToList(); @@ -69,7 +81,7 @@ public override async Task HandleItemAsync(WorkItemContext context) int eventsPerProject = eventCount / projectList.Count; int remainder = eventCount % projectList.Count; int totalProcessed = 0; - const int batchSize = 50; + const int batchSize = 100; for (int p = 0; p < projectList.Count; p++) { @@ -81,14 +93,13 @@ public override async Task HandleItemAsync(WorkItemContext context) var events = generator.Generate(organization.Id, project.Id, projectEventCount, minDate, utcNow); - for (int i = 0; i < events.Count; i += batchSize) + foreach (var batch in events.Chunk(batchSize)) { if (context.CancellationToken.IsCancellationRequested) break; - var batch = events.Skip(i).Take(batchSize).ToList(); await _eventPipeline.RunAsync(batch, organization, project); - totalProcessed += batch.Count; + totalProcessed += batch.Length; int percentage = (int)Math.Min(99, totalProcessed * 100.0 / eventCount); await context.ReportProgressAsync(percentage, $"Processed {totalProcessed}/{eventCount} events"); @@ -98,4 +109,51 @@ public override async Task HandleItemAsync(WorkItemContext context) await context.ReportProgressAsync(100, $"Generated {totalProcessed} sample events across {projectList.Count} projects"); Log.LogInformation("Generated {TotalEvents} sample events across {ProjectCount} projects", totalProcessed, projectList.Count); } + + private async Task GenerateProjectSampleEventsAsync(WorkItemContext context, RandomEventGenerator generator, GenerateSampleEventsWorkItem workItem, int eventCount, DateTime minDate, DateTime utcNow) + { + if (String.IsNullOrEmpty(workItem.OrganizationId) || String.IsNullOrEmpty(workItem.ProjectId)) + { + Log.LogWarning("Unable to generate project sample events because organization id or project id was not specified"); + return; + } + + var organization = await _organizationRepository.GetByIdAsync(workItem.OrganizationId); + if (organization is null) + { + Log.LogWarning("Organization {OrganizationId} not found when generating sample events", workItem.OrganizationId); + return; + } + + var project = await _projectRepository.GetByIdAsync(workItem.ProjectId); + if (project is null || !String.Equals(project.OrganizationId, organization.Id)) + { + Log.LogWarning("Project {ProjectId} not found in organization {OrganizationId} when generating sample events", workItem.ProjectId, workItem.OrganizationId); + return; + } + + int totalProcessed = 0; + const int batchSize = 100; + var events = generator.Generate(organization.Id, project.Id, eventCount, minDate, utcNow); + + foreach (var batch in events.Chunk(batchSize)) + { + if (context.CancellationToken.IsCancellationRequested) + break; + + await _eventPipeline.RunAsync(batch, organization, project); + totalProcessed += batch.Length; + + int percentage = (int)Math.Min(99, totalProcessed * 100.0 / eventCount); + await context.ReportProgressAsync(percentage, $"Processed {totalProcessed}/{eventCount} events"); + } + + await context.ReportProgressAsync(100, $"Generated {totalProcessed} sample events for project {project.Id}"); + Log.LogInformation("Generated {TotalEvents} sample events for project {ProjectId}", totalProcessed, project.Id); + } + + private static bool IsProjectScoped(GenerateSampleEventsWorkItem workItem) + { + return !String.IsNullOrEmpty(workItem.OrganizationId) && !String.IsNullOrEmpty(workItem.ProjectId); + } } diff --git a/src/Exceptionless.Core/Jobs/WorkItemHandlers/RemoveStacksWorkItemHandler.cs b/src/Exceptionless.Core/Jobs/WorkItemHandlers/RemoveStacksWorkItemHandler.cs index 40de299f71..e816077a8d 100644 --- a/src/Exceptionless.Core/Jobs/WorkItemHandlers/RemoveStacksWorkItemHandler.cs +++ b/src/Exceptionless.Core/Jobs/WorkItemHandlers/RemoveStacksWorkItemHandler.cs @@ -1,5 +1,6 @@ using Exceptionless.Core.Models.WorkItems; using Exceptionless.Core.Repositories; +using Exceptionless.Core.Repositories.Queries; using Foundatio.Caching; using Foundatio.Jobs; using Foundatio.Lock; @@ -35,7 +36,7 @@ public override async Task HandleItemAsync(WorkItemContext context) Log.LogInformation("Received remove stacks work item for project: {ProjectId}", wi.ProjectId); await context.ReportProgressAsync(0, "Starting soft deleting of stacks..."); long deleted = await _stackRepository.SoftDeleteByProjectIdAsync(wi.OrganizationId, wi.ProjectId); - await _cacheClient.RemoveByPrefixAsync(String.Concat("stack-filter:", wi.OrganizationId, ":", wi.ProjectId)); + await _cacheClient.RemoveByPrefixAsync(EventStackFilterQueryBuilder.GetScopedCachePrefix(wi.OrganizationId, wi.ProjectId)); await context.ReportProgressAsync(100, $"Stacks soft deleted: {deleted}"); } } diff --git a/src/Exceptionless.Core/Jobs/WorkItemHandlers/ResetProjectDataWorkItemHandler.cs b/src/Exceptionless.Core/Jobs/WorkItemHandlers/ResetProjectDataWorkItemHandler.cs new file mode 100644 index 0000000000..3bed9eaeb3 --- /dev/null +++ b/src/Exceptionless.Core/Jobs/WorkItemHandlers/ResetProjectDataWorkItemHandler.cs @@ -0,0 +1,51 @@ +using Exceptionless.Core.Models.WorkItems; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Repositories.Queries; +using Foundatio.Caching; +using Foundatio.Jobs; +using Foundatio.Lock; +using Microsoft.Extensions.Logging; + +namespace Exceptionless.Core.Jobs.WorkItemHandlers; + +public class ResetProjectDataWorkItemHandler : WorkItemHandlerBase +{ + private readonly IEventRepository _eventRepository; + private readonly IStackRepository _stackRepository; + private readonly ICacheClient _cacheClient; + private readonly ILockProvider _lockProvider; + + public ResetProjectDataWorkItemHandler(IEventRepository eventRepository, IStackRepository stackRepository, ICacheClient cacheClient, ILockProvider lockProvider, ILoggerFactory loggerFactory) : base(loggerFactory) + { + _eventRepository = eventRepository; + _stackRepository = stackRepository; + _cacheClient = cacheClient; + _lockProvider = lockProvider; + } + + public override Task GetWorkItemLockAsync(object workItem, CancellationToken cancellationToken = default) + { + string cacheKey = $"{nameof(ResetProjectDataWorkItemHandler)}:{((ResetProjectDataWorkItem)workItem).ProjectId}"; + return _lockProvider.TryAcquireAsync(cacheKey, TimeSpan.FromMinutes(15), cancellationToken); + } + + public override async Task HandleItemAsync(WorkItemContext context) + { + var workItem = context.GetData()!; + + using (Log.BeginScope(new ExceptionlessState().Organization(workItem.OrganizationId).Project(workItem.ProjectId))) + { + Log.LogInformation("Received reset project data work item for project: {ProjectId}", workItem.ProjectId); + await context.ReportProgressAsync(0, "Starting project data reset..."); + + long removedEvents = await _eventRepository.RemoveAllByProjectIdAsync(workItem.OrganizationId, workItem.ProjectId); + await context.ReportProgressAsync(50, $"Events removed: {removedEvents}"); + + long removedStacks = await _stackRepository.RemoveAllByProjectIdAsync(workItem.OrganizationId, workItem.ProjectId); + await _cacheClient.RemoveByPrefixAsync(EventStackFilterQueryBuilder.GetScopedCachePrefix(workItem.OrganizationId, workItem.ProjectId)); + + await context.ReportProgressAsync(100, $"Events removed: {removedEvents}, stacks removed: {removedStacks}"); + Log.LogInformation("Reset project data for project {ProjectId}. Events removed: {RemovedEvents}, stacks removed: {RemovedStacks}", workItem.ProjectId, removedEvents, removedStacks); + } + } +} diff --git a/src/Exceptionless.Core/Models/WorkItems/GenerateSampleEventsWorkItem.cs b/src/Exceptionless.Core/Models/WorkItems/GenerateSampleEventsWorkItem.cs index 06ec8c4dde..a25dbd94de 100644 --- a/src/Exceptionless.Core/Models/WorkItems/GenerateSampleEventsWorkItem.cs +++ b/src/Exceptionless.Core/Models/WorkItems/GenerateSampleEventsWorkItem.cs @@ -2,6 +2,8 @@ namespace Exceptionless.Core.Models.WorkItems; public record GenerateSampleEventsWorkItem { + public string? OrganizationId { get; init; } + public string? ProjectId { get; init; } public int EventCount { get; init; } = 100; public int DaysBack { get; init; } = 7; } diff --git a/src/Exceptionless.Core/Models/WorkItems/ResetProjectDataWorkItem.cs b/src/Exceptionless.Core/Models/WorkItems/ResetProjectDataWorkItem.cs new file mode 100644 index 0000000000..4d4410c6c0 --- /dev/null +++ b/src/Exceptionless.Core/Models/WorkItems/ResetProjectDataWorkItem.cs @@ -0,0 +1,7 @@ +namespace Exceptionless.Core.Models.WorkItems; + +public record ResetProjectDataWorkItem +{ + public required string OrganizationId { get; init; } + public required string ProjectId { get; init; } +} diff --git a/src/Exceptionless.Core/Repositories/Queries/EventStackFilterQuery.cs b/src/Exceptionless.Core/Repositories/Queries/EventStackFilterQuery.cs index 34fc9800a3..4d78edd8e1 100644 --- a/src/Exceptionless.Core/Repositories/Queries/EventStackFilterQuery.cs +++ b/src/Exceptionless.Core/Repositories/Queries/EventStackFilterQuery.cs @@ -56,6 +56,8 @@ namespace Exceptionless.Core.Repositories.Queries { public class EventStackFilterQueryBuilder : IElasticQueryBuilder { + public const string CacheScope = "stack-filter"; + private readonly IStackRepository _stackRepository; private readonly ILogger _logger; private readonly Field _inferredEventDateField; @@ -66,13 +68,18 @@ public class EventStackFilterQueryBuilder : IElasticQueryBuilder public EventStackFilterQueryBuilder(IStackRepository stackRepository, ICacheClient cacheClient, ILoggerFactory loggerFactory) { _stackRepository = stackRepository; - _cacheClient = new ScopedCacheClient(cacheClient, "stack-filter"); + _cacheClient = new ScopedCacheClient(cacheClient, CacheScope); _logger = loggerFactory.CreateLogger(); _inferredEventDateField = Infer.Field(f => f.Date); _inferredStackLastOccurrenceField = Infer.Field(f => f.LastOccurrence); _eventStackFilter = new EventStackFilter(); } + public static string GetScopedCachePrefix(string organizationId, string projectId) + { + return String.Concat(CacheScope, ":", organizationId, ":", projectId); + } + public async Task BuildAsync(QueryBuilderContext ctx) where T : class, new() { if (!ctx.Source.ShouldEnforceEventStackFilter()) diff --git a/src/Exceptionless.Core/Utility/SampleDataService.cs b/src/Exceptionless.Core/Utility/SampleDataService.cs index c778b21fe6..897a2e92cc 100644 --- a/src/Exceptionless.Core/Utility/SampleDataService.cs +++ b/src/Exceptionless.Core/Utility/SampleDataService.cs @@ -286,4 +286,18 @@ await _workItemQueue.EnqueueAsync(new GenerateSampleEventsWorkItem }); _logger.LogInformation("Enqueued sample event generation: {EventCount} events over {DaysBack} days", eventCount, daysBack); } + + public async Task EnqueueSampleEventsAsync(string organizationId, string projectId, int eventCount = 100, int daysBack = 7) + { + string workItemId = await _workItemQueue.EnqueueAsync(new GenerateSampleEventsWorkItem + { + OrganizationId = organizationId, + ProjectId = projectId, + EventCount = eventCount, + DaysBack = daysBack + }); + + _logger.LogInformation("Enqueued sample event generation for project {ProjectId}: {EventCount} events over {DaysBack} days", projectId, eventCount, daysBack); + return workItemId; + } } diff --git a/src/Exceptionless.Web/ClientApp.angular/app/project/configure-controller.js b/src/Exceptionless.Web/ClientApp.angular/app/project/configure-controller.js index fbc2dde362..b10b60fe1b 100644 --- a/src/Exceptionless.Web/ClientApp.angular/app/project/configure-controller.js +++ b/src/Exceptionless.Web/ClientApp.angular/app/project/configure-controller.js @@ -168,6 +168,32 @@ $state.go("app.project-frequent", { projectId: vm._projectId }); } + function generateSampleData() { + if (vm.isGeneratingSampleData) { + return; + } + + function onSuccess() { + notificationService.success( + translateService.T("Sample data generation has been queued. Events will appear shortly.") + ); + } + + function onFailure() { + notificationService.error( + translateService.T("An error occurred while generating sample data for your project.") + ); + } + + vm.isGeneratingSampleData = true; + return projectService + .generateSampleData(vm._projectId) + .then(onSuccess, onFailure) + .finally(function () { + vm.isGeneratingSampleData = false; + }); + } + this.$onInit = function $onInit() { vm._projectId = $stateParams.id; vm._canRedirect = $stateParams.redirect === "true"; @@ -177,9 +203,11 @@ vm.copyCommandLineCode = copyCommandLineCode; vm.copied = copied; vm.currentProjectType = {}; + vm.generateSampleData = generateSampleData; vm.isBashShell = isBashShell; vm.isCommandLine = isCommandLine; vm.isDotNet = isDotNet; + vm.isGeneratingSampleData = false; vm.isJavaScript = isJavaScript; vm.isNode = isNode; vm.navigateToDashboard = navigateToDashboard; diff --git a/src/Exceptionless.Web/ClientApp.angular/app/project/configure.tpl.html b/src/Exceptionless.Web/ClientApp.angular/app/project/configure.tpl.html index e6c6060819..c9fb2d6373 100644 --- a/src/Exceptionless.Web/ClientApp.angular/app/project/configure.tpl.html +++ b/src/Exceptionless.Web/ClientApp.angular/app/project/configure.tpl.html @@ -182,6 +182,19 @@