Add project sample data generation

src/Exceptionless.Core/Bootstrapper.cs

@@ -107,6 +107,7 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO
             handlers.Register<ReindexWorkItem>(s.GetRequiredService<ReindexWorkItemHandler>);
             handlers.Register<RemoveBotEventsWorkItem>(s.GetRequiredService<RemoveBotEventsWorkItemHandler>);
             handlers.Register<RemoveStacksWorkItem>(s.GetRequiredService<RemoveStacksWorkItemHandler>);
+            handlers.Register<ResetProjectDataWorkItem>(s.GetRequiredService<ResetProjectDataWorkItemHandler>);
             handlers.Register<SetLocationFromGeoWorkItem>(s.GetRequiredService<SetLocationFromGeoWorkItemHandler>);
             handlers.Register<SetProjectIsConfiguredWorkItem>(s.GetRequiredService<SetProjectIsConfiguredWorkItemHandler>);
             handlers.Register<UpdateProjectNotificationSettingsWorkItem>(s.GetRequiredService<UpdateProjectNotificationSettingsWorkItemHandler>);

src/Exceptionless.Core/Jobs/CleanupDataJob.cs

@@ -2,6 +2,7 @@
 using Exceptionless.Core.Extensions;
 using Exceptionless.Core.Models;
 using Exceptionless.Core.Repositories;
+using Exceptionless.Core.Repositories.Queries;
 using Exceptionless.Core.Services;
 using Exceptionless.DateTimeExtensions;
 using Foundatio.Caching;
@@ -220,7 +221,7 @@ private async Task RemoveStacksAsync(IReadOnlyCollection<Stack> stacks, JobConte
         long removedEvents = await _eventRepository.RemoveAllByStackIdsAsync(stackIds);
         await _stackRepository.RemoveAsync(stacks);
         foreach (var orgGroup in stacks.GroupBy(s => (s.OrganizationId, s.ProjectId)))
-            await _cacheClient.RemoveByPrefixAsync(String.Concat("stack-filter:", orgGroup.Key.OrganizationId, ":", orgGroup.Key.ProjectId));
+            await _cacheClient.RemoveByPrefixAsync(EventStackFilterQueryBuilder.GetScopedCachePrefix(orgGroup.Key.OrganizationId, orgGroup.Key.ProjectId));
 
         _logger.RemoveStacksComplete(stackIds.Length, removedEvents);
     }

src/Exceptionless.Core/Jobs/WorkItemHandlers/GenerateSampleEventsWorkItemHandler.cs

@@ -35,21 +35,33 @@ public GenerateSampleEventsWorkItemHandler(
 
     public override Task<ILock?> GetWorkItemLockAsync(object workItem, CancellationToken cancellationToken = default)
     {
-        return _lockProvider.TryAcquireAsync(nameof(GenerateSampleEventsWorkItemHandler), TimeSpan.FromMinutes(30), cancellationToken);
+        var generateSampleEventsWorkItem = (GenerateSampleEventsWorkItem)workItem;
+        string cacheKey = IsProjectScoped(generateSampleEventsWorkItem)
+            ? $"{nameof(GenerateSampleEventsWorkItemHandler)}:{generateSampleEventsWorkItem.ProjectId}"
+            : nameof(GenerateSampleEventsWorkItemHandler);
+
+        return _lockProvider.TryAcquireAsync(cacheKey, TimeSpan.FromMinutes(30), cancellationToken);
     }
 
     public override async Task HandleItemAsync(WorkItemContext context)
     {
         var workItem = context.GetData<GenerateSampleEventsWorkItem>()!;
         int eventCount = Math.Clamp(workItem.EventCount, 1, 10000);
         int daysBack = Math.Clamp(workItem.DaysBack, 1, 365);
+        int acceptedDaysBack = Math.Min(daysBack, 3);
 
-        Log.LogInformation("Generating {EventCount} sample events over {DaysBack} days", eventCount, daysBack);
-        await context.ReportProgressAsync(0, $"Generating {eventCount} sample events");
+        Log.LogInformation("Generating {EventCount} sample events over {DaysBack} days", eventCount, acceptedDaysBack);
+        await context.ReportProgressAsync(0, $"Generating {eventCount} sample events over {acceptedDaysBack} days");
 
         var generator = new RandomEventGenerator(_timeProvider);
         var utcNow = _timeProvider.GetUtcNow().UtcDateTime;
-        var minDate = utcNow.AddDays(-daysBack);
+        var minDate = utcNow.AddDays(-acceptedDaysBack);
+
+        if (IsProjectScoped(workItem))
+        {
+            await GenerateProjectSampleEventsAsync(context, generator, workItem, eventCount, minDate, utcNow);
+            return;
+        }
 
         var projectResults = await _projectRepository.GetByOrganizationIdAsync(SampleDataService.TEST_ORG_ID);
         var projectList = projectResults.Documents.ToList();
@@ -69,7 +81,7 @@ public override async Task HandleItemAsync(WorkItemContext context)
         int eventsPerProject = eventCount / projectList.Count;
         int remainder = eventCount % projectList.Count;
         int totalProcessed = 0;
-        const int batchSize = 50;
+        const int batchSize = 100;
 
         for (int p = 0; p < projectList.Count; p++)
         {
@@ -81,14 +93,13 @@ public override async Task HandleItemAsync(WorkItemContext context)
 
             var events = generator.Generate(organization.Id, project.Id, projectEventCount, minDate, utcNow);
 
-            for (int i = 0; i < events.Count; i += batchSize)
+            foreach (var batch in events.Chunk(batchSize))
             {
                 if (context.CancellationToken.IsCancellationRequested)
                     break;
 
-                var batch = events.Skip(i).Take(batchSize).ToList();
                 await _eventPipeline.RunAsync(batch, organization, project);
-                totalProcessed += batch.Count;
+                totalProcessed += batch.Length;
 
                 int percentage = (int)Math.Min(99, totalProcessed * 100.0 / eventCount);
                 await context.ReportProgressAsync(percentage, $"Processed {totalProcessed}/{eventCount} events");
@@ -98,4 +109,51 @@ public override async Task HandleItemAsync(WorkItemContext context)
         await context.ReportProgressAsync(100, $"Generated {totalProcessed} sample events across {projectList.Count} projects");
         Log.LogInformation("Generated {TotalEvents} sample events across {ProjectCount} projects", totalProcessed, projectList.Count);
     }
+
+    private async Task GenerateProjectSampleEventsAsync(WorkItemContext context, RandomEventGenerator generator, GenerateSampleEventsWorkItem workItem, int eventCount, DateTime minDate, DateTime utcNow)
+    {
+        if (String.IsNullOrEmpty(workItem.OrganizationId) || String.IsNullOrEmpty(workItem.ProjectId))
+        {
+            Log.LogWarning("Unable to generate project sample events because organization id or project id was not specified");
+            return;
+        }
+
+        var organization = await _organizationRepository.GetByIdAsync(workItem.OrganizationId);
+        if (organization is null)
+        {
+            Log.LogWarning("Organization {OrganizationId} not found when generating sample events", workItem.OrganizationId);
+            return;
+        }
+
+        var project = await _projectRepository.GetByIdAsync(workItem.ProjectId);
+        if (project is null || !String.Equals(project.OrganizationId, organization.Id))
+        {
+            Log.LogWarning("Project {ProjectId} not found in organization {OrganizationId} when generating sample events", workItem.ProjectId, workItem.OrganizationId);
+            return;
+        }
+
+        int totalProcessed = 0;
+        const int batchSize = 100;
+        var events = generator.Generate(organization.Id, project.Id, eventCount, minDate, utcNow);
+
+        foreach (var batch in events.Chunk(batchSize))
+        {
+            if (context.CancellationToken.IsCancellationRequested)
+                break;
+
+            await _eventPipeline.RunAsync(batch, organization, project);
+            totalProcessed += batch.Length;
+
+            int percentage = (int)Math.Min(99, totalProcessed * 100.0 / eventCount);
+            await context.ReportProgressAsync(percentage, $"Processed {totalProcessed}/{eventCount} events");
+        }
+
+        await context.ReportProgressAsync(100, $"Generated {totalProcessed} sample events for project {project.Id}");
+        Log.LogInformation("Generated {TotalEvents} sample events for project {ProjectId}", totalProcessed, project.Id);
+    }
+
+    private static bool IsProjectScoped(GenerateSampleEventsWorkItem workItem)
+    {
+        return !String.IsNullOrEmpty(workItem.OrganizationId) && !String.IsNullOrEmpty(workItem.ProjectId);
+    }
 }

src/Exceptionless.Core/Jobs/WorkItemHandlers/RemoveStacksWorkItemHandler.cs

@@ -1,5 +1,6 @@
 using Exceptionless.Core.Models.WorkItems;
 using Exceptionless.Core.Repositories;
+using Exceptionless.Core.Repositories.Queries;
 using Foundatio.Caching;
 using Foundatio.Jobs;
 using Foundatio.Lock;
@@ -35,7 +36,7 @@ public override async Task HandleItemAsync(WorkItemContext context)
             Log.LogInformation("Received remove stacks work item for project: {ProjectId}", wi.ProjectId);
             await context.ReportProgressAsync(0, "Starting soft deleting of stacks...");
             long deleted = await _stackRepository.SoftDeleteByProjectIdAsync(wi.OrganizationId, wi.ProjectId);
-            await _cacheClient.RemoveByPrefixAsync(String.Concat("stack-filter:", wi.OrganizationId, ":", wi.ProjectId));
+            await _cacheClient.RemoveByPrefixAsync(EventStackFilterQueryBuilder.GetScopedCachePrefix(wi.OrganizationId, wi.ProjectId));
             await context.ReportProgressAsync(100, $"Stacks soft deleted: {deleted}");
         }
     }

src/Exceptionless.Core/Jobs/WorkItemHandlers/ResetProjectDataWorkItemHandler.cs

@@ -0,0 +1,51 @@
+using Exceptionless.Core.Models.WorkItems;
+using Exceptionless.Core.Repositories;
+using Exceptionless.Core.Repositories.Queries;
+using Foundatio.Caching;
+using Foundatio.Jobs;
+using Foundatio.Lock;
+using Microsoft.Extensions.Logging;
+
+namespace Exceptionless.Core.Jobs.WorkItemHandlers;
+
+public class ResetProjectDataWorkItemHandler : WorkItemHandlerBase
+{
+    private readonly IEventRepository _eventRepository;
+    private readonly IStackRepository _stackRepository;
+    private readonly ICacheClient _cacheClient;
+    private readonly ILockProvider _lockProvider;
+
+    public ResetProjectDataWorkItemHandler(IEventRepository eventRepository, IStackRepository stackRepository, ICacheClient cacheClient, ILockProvider lockProvider, ILoggerFactory loggerFactory) : base(loggerFactory)
+    {
+        _eventRepository = eventRepository;
+        _stackRepository = stackRepository;
+        _cacheClient = cacheClient;
+        _lockProvider = lockProvider;
+    }
+
+    public override Task<ILock?> GetWorkItemLockAsync(object workItem, CancellationToken cancellationToken = default)
+    {
+        string cacheKey = $"{nameof(ResetProjectDataWorkItemHandler)}:{((ResetProjectDataWorkItem)workItem).ProjectId}";
+        return _lockProvider.TryAcquireAsync(cacheKey, TimeSpan.FromMinutes(15), cancellationToken);
+    }
+
+    public override async Task HandleItemAsync(WorkItemContext context)
+    {
+        var workItem = context.GetData<ResetProjectDataWorkItem>()!;
+
+        using (Log.BeginScope(new ExceptionlessState().Organization(workItem.OrganizationId).Project(workItem.ProjectId)))
+        {
+            Log.LogInformation("Received reset project data work item for project: {ProjectId}", workItem.ProjectId);
+            await context.ReportProgressAsync(0, "Starting project data reset...");
+
+            long removedEvents = await _eventRepository.RemoveAllByProjectIdAsync(workItem.OrganizationId, workItem.ProjectId);
+            await context.ReportProgressAsync(50, $"Events removed: {removedEvents}");
+
+            long removedStacks = await _stackRepository.RemoveAllByProjectIdAsync(workItem.OrganizationId, workItem.ProjectId);
+            await _cacheClient.RemoveByPrefixAsync(EventStackFilterQueryBuilder.GetScopedCachePrefix(workItem.OrganizationId, workItem.ProjectId));
+
+            await context.ReportProgressAsync(100, $"Events removed: {removedEvents}, stacks removed: {removedStacks}");
+            Log.LogInformation("Reset project data for project {ProjectId}. Events removed: {RemovedEvents}, stacks removed: {RemovedStacks}", workItem.ProjectId, removedEvents, removedStacks);
+        }
+    }
+}

src/Exceptionless.Core/Models/WorkItems/GenerateSampleEventsWorkItem.cs

@@ -2,6 +2,8 @@ namespace Exceptionless.Core.Models.WorkItems;
 
 public record GenerateSampleEventsWorkItem
 {
+    public string? OrganizationId { get; init; }
+    public string? ProjectId { get; init; }
     public int EventCount { get; init; } = 100;
     public int DaysBack { get; init; } = 7;
 }

src/Exceptionless.Core/Models/WorkItems/ResetProjectDataWorkItem.cs

@@ -0,0 +1,7 @@
+namespace Exceptionless.Core.Models.WorkItems;
+
+public record ResetProjectDataWorkItem
+{
+    public required string OrganizationId { get; init; }
+    public required string ProjectId { get; init; }
+}

src/Exceptionless.Core/Repositories/Queries/EventStackFilterQuery.cs

@@ -56,6 +56,8 @@ namespace Exceptionless.Core.Repositories.Queries
 {
     public class EventStackFilterQueryBuilder : IElasticQueryBuilder
     {
+        public const string CacheScope = "stack-filter";
+
         private readonly IStackRepository _stackRepository;
         private readonly ILogger _logger;
         private readonly Field _inferredEventDateField;
@@ -66,13 +68,18 @@ public class EventStackFilterQueryBuilder : IElasticQueryBuilder
         public EventStackFilterQueryBuilder(IStackRepository stackRepository, ICacheClient cacheClient, ILoggerFactory loggerFactory)
         {
             _stackRepository = stackRepository;
-            _cacheClient = new ScopedCacheClient(cacheClient, "stack-filter");
+            _cacheClient = new ScopedCacheClient(cacheClient, CacheScope);
             _logger = loggerFactory.CreateLogger<EventStackFilterQueryBuilder>();
             _inferredEventDateField = Infer.Field<PersistentEvent>(f => f.Date);
             _inferredStackLastOccurrenceField = Infer.Field<Stack>(f => f.LastOccurrence);
             _eventStackFilter = new EventStackFilter();
         }
 
+        public static string GetScopedCachePrefix(string organizationId, string projectId)
+        {
+            return String.Concat(CacheScope, ":", organizationId, ":", projectId);
+        }
+
         public async Task BuildAsync<T>(QueryBuilderContext<T> ctx) where T : class, new()
         {
             if (!ctx.Source.ShouldEnforceEventStackFilter())

src/Exceptionless.Core/Utility/SampleDataService.cs

@@ -286,4 +286,18 @@ await _workItemQueue.EnqueueAsync(new GenerateSampleEventsWorkItem
         });
         _logger.LogInformation("Enqueued sample event generation: {EventCount} events over {DaysBack} days", eventCount, daysBack);
     }
+
+    public async Task<string> EnqueueSampleEventsAsync(string organizationId, string projectId, int eventCount = 100, int daysBack = 7)
+    {
+        string workItemId = await _workItemQueue.EnqueueAsync(new GenerateSampleEventsWorkItem
+        {
+            OrganizationId = organizationId,
+            ProjectId = projectId,
+            EventCount = eventCount,
+            DaysBack = daysBack
+        });
+
+        _logger.LogInformation("Enqueued sample event generation for project {ProjectId}: {EventCount} events over {DaysBack} days", projectId, eventCount, daysBack);
+        return workItemId;
+    }
 }

src/Exceptionless.Web/ClientApp.angular/app/project/configure-controller.js

@@ -168,6 +168,32 @@
                     $state.go("app.project-frequent", { projectId: vm._projectId });
                 }
 
+                function generateSampleData() {
+                    if (vm.isGeneratingSampleData) {
+                        return;
+                    }
+
+                    function onSuccess() {
+                        notificationService.success(
+                            translateService.T("Sample data generation has been queued. Events will appear shortly.")
+                        );
+                    }
+
+                    function onFailure() {
+                        notificationService.error(
+                            translateService.T("An error occurred while generating sample data for your project.")
+                        );
+                    }
+
+                    vm.isGeneratingSampleData = true;
+                    return projectService
+                        .generateSampleData(vm._projectId)
+                        .then(onSuccess, onFailure)
+                        .finally(function () {
+                            vm.isGeneratingSampleData = false;
+                        });
+                }
+
                 this.$onInit = function $onInit() {
                     vm._projectId = $stateParams.id;
                     vm._canRedirect = $stateParams.redirect === "true";
@@ -177,9 +203,11 @@
                     vm.copyCommandLineCode = copyCommandLineCode;
                     vm.copied = copied;
                     vm.currentProjectType = {};
+                    vm.generateSampleData = generateSampleData;
                     vm.isBashShell = isBashShell;
                     vm.isCommandLine = isCommandLine;
                     vm.isDotNet = isDotNet;
+                    vm.isGeneratingSampleData = false;
                     vm.isJavaScript = isJavaScript;
                     vm.isNode = isNode;
                     vm.navigateToDashboard = navigateToDashboard;

src/Exceptionless.Web/ClientApp.angular/app/project/configure.tpl.html

@@ -182,6 +182,19 @@
                 </div>
                 <footer class="panel-footer">
                     <div class="pull-right">
+                        <button
+                            type="button"
+                            class="btn btn-primary m-r-xs"
+                            ng-click="vm.generateSampleData()"
+                            ng-disabled="vm.isGeneratingSampleData"
+                        >
+                            <i
+                                class="fa fa-fw"
+                                ng-class="vm.isGeneratingSampleData ? 'fa-spinner fa-spin' : 'fa-database'"
+                            ></i>
+                            <span ng-if="!vm.isGeneratingSampleData">{{::'Generate Sample Data' | translate}}</span>
+                            <span ng-if="vm.isGeneratingSampleData">{{::'Generating Sample Data...' | translate}}</span>
+                        </button>
                         <a
                             ui-sref="app.project-frequent({ projectId: vm.project.id })"
                             class="btn btn-default"

src/Exceptionless.Web/ClientApp.angular/components/project/project-service.js

@@ -78,6 +78,10 @@
                 return _cachedRestangular.one("users", userId).one("projects", id).one("notifications").get();
             }
 
+            function generateSampleData(id) {
+                return Restangular.one("projects", id).post("sample-data");
+            }
+
             function getIntegrationNotificationSettings(id, integration) {
                 return _cachedRestangular.one("projects", id).one(integration, "notifications").get();
             }
@@ -113,7 +117,7 @@
             }
 
             function resetData(id) {
-                return Restangular.one("projects", id).one("reset-data").get();
+                return Restangular.one("projects", id).post("reset-data");
             }
 
             function update(id, project) {
@@ -145,6 +149,7 @@
                 getByOrganizationId: getByOrganizationId,
                 getConfig: getConfig,
                 getNotificationSettings: getNotificationSettings,
+                generateSampleData: generateSampleData,
                 getIntegrationNotificationSettings: getIntegrationNotificationSettings,
                 isNameAvailable: isNameAvailable,
                 promoteTab: promoteTab,

src/Exceptionless.Web/ClientApp.angular/lang/en-us.json

@@ -376,6 +376,10 @@
     "An error occurred while trying to remove the project.": "An error occurred while trying to remove the project.",
     "Download_Configure_Project": "Download & Configure Project {{projectName}} Client",
     "The Exceptionless client can be integrated into your project in just a few easy steps.": "The Exceptionless client can be integrated into your project in just a few easy steps.",
+    "Generate Sample Data": "Generate Sample Data",
+    "Generating Sample Data...": "Generating Sample Data...",
+    "Sample data generation has been queued. Events will appear shortly.": "Sample data generation has been queued. Events will appear shortly.",
+    "An error occurred while generating sample data for your project.": "An error occurred while generating sample data for your project.",
     "Select your project type:": "Select your project type:",
     "Please select a project type": "Please select a project type",
     "Execute the following in your shell:": "Execute the following in your shell:",