Form1.cs

using System;
using System.Collections.Generic;
using System.Drawing;
using System.Threading;
using System.Threading.Tasks;
using System.Windows.Forms;

namespace WebToolsDataMonitor
{
    public partial class Form1 : Form
    {
        private ListView engineListView;
        private Button btnStartStop;

        // 🔥 NEW Toggle Controls
        private CheckBox chkParallelTools;
        private CheckBox chkParallelPages;

        // Continuous run loop controls
        private NumericUpDown numIntervalSeconds;
        private Button btnStartLoop;

        private Form2 monitoringDashboard = null;
        private CancellationTokenSource _cts = null;
        private CancellationTokenSource _loopCts = null;
        private Dictionary<string, ListViewItem> _engineRows = new Dictionary<string, ListViewItem>(StringComparer.OrdinalIgnoreCase);

        public Form1()
        {
            InitializeComponent();
            SetupControlBridgeLayout();
        }

        private void SetupControlBridgeLayout()
        {
            this.Text = "Automation Engine Command Panel (Form1)";
            this.Size = new Size(600, 650);
            this.StartPosition = FormStartPosition.CenterScreen;

            engineListView = new ListView
            {
                Location = new Point(15, 15),
                Size = new Size(550, 140),
                View = View.Details,
                FullRowSelect = true,
                CheckBoxes = true,
                GridLines = true
            };
            engineListView.Columns.Add("Run", 50);
            engineListView.Columns.Add("Engine Tool Name", 150);
            engineListView.Columns.Add("Execution State Status", 200);
            engineListView.Columns.Add("Active Workers", 100);

            foreach (var tool in ToolRepository.Instance.GetAllTools())
            {
                RegisterOrUpdateEngineState(tool.ToolName, "Idle Ready", 0, true);
            }

            btnStartStop = new Button
            {
                Text = "START SCRAPER ENGINE",
                Location = new Point(15, 165),
                Size = new Size(550, 40),
                BackColor = Color.LightGreen,
                Font = new Font(this.Font, FontStyle.Bold)
            };
            btnStartStop.Click += BtnStartStop_Click;

            // 🔥 NEW Checkbox Toggles Instantiation - initial state restored from app_config.yaml
            var uiState = AppConfig.Instance.Data.Ui;

            chkParallelTools = new CheckBox
            {
                Text = "Execute Selected Tools in Parallel",
                Location = new Point(15, 215),
                Size = new Size(250, 20),
                Checked = uiState.ParallelTools
            };
            chkParallelTools.CheckedChanged += (s, e) =>
            {
                AppConfig.Instance.Data.Ui.ParallelTools = chkParallelTools.Checked;
                AppConfig.Instance.Save();
            };

            chkParallelPages = new CheckBox
            {
                Text = "Execute Tool Pages in Parallel (Shared Session Cache)",
                Location = new Point(275, 215),
                Size = new Size(290, 20),
                Checked = uiState.ParallelPages
            };
            chkParallelPages.CheckedChanged += (s, e) =>
            {
                AppConfig.Instance.Data.Ui.ParallelPages = chkParallelPages.Checked;
                AppConfig.Instance.Save();
            };

            // 🔥 NEW Continuous Run Loop Controls
            Label lblInterval = new Label
            {
                Text = "Continuous Run Interval (seconds):",
                Location = new Point(15, 243),
                Size = new Size(200, 20)
            };

            numIntervalSeconds = new NumericUpDown
            {
                Location = new Point(220, 240),
                Size = new Size(70, 22),
                Minimum = 1,
                Maximum = 86400
            };
            numIntervalSeconds.Value = Math.Max(numIntervalSeconds.Minimum, Math.Min(numIntervalSeconds.Maximum, uiState.IntervalSeconds));
            numIntervalSeconds.ValueChanged += (s, e) =>
            {
                AppConfig.Instance.Data.Ui.IntervalSeconds = (int)numIntervalSeconds.Value;
                AppConfig.Instance.Save();
            };

            btnStartLoop = new Button
            {
                Text = "START CONTINUOUS RUN",
                Location = new Point(15, 268),
                Size = new Size(550, 36),
                BackColor = Color.LightGreen,
                Font = new Font(this.Font, FontStyle.Bold)
            };
            btnStartLoop.Click += BtnStartLoop_Click;

            // Shifted down to make room for the continuous run controls above
            StatusBox.Location = new Point(15, 314);
            StatusBox.Size = new Size(550, 270);
            StatusBox.Multiline = true;
            StatusBox.ScrollBars = ScrollBars.Vertical;
            StatusBox.ReadOnly = true;
            StatusBox.BackColor = Color.Black;
            StatusBox.ForeColor = Color.Lime;

            this.Controls.Add(engineListView);
            this.Controls.Add(btnStartStop);
            this.Controls.Add(chkParallelTools);
            this.Controls.Add(chkParallelPages);
            this.Controls.Add(lblInterval);
            this.Controls.Add(numIntervalSeconds);
            this.Controls.Add(btnStartLoop);
        }

        public void RegisterOrUpdateEngineState(string toolName, string runningStatus, int threadCount, bool? setCheckedState = null)
        {
            if (this.InvokeRequired)
            {
                this.BeginInvoke((MethodInvoker)(() => RegisterOrUpdateEngineState(toolName, runningStatus, threadCount, setCheckedState)));
                return;
            }

            if (_engineRows.TryGetValue(toolName, out var targetRow))
            {
                targetRow.SubItems[2].Text = runningStatus;
                targetRow.SubItems[3].Text = threadCount.ToString();
                if (setCheckedState.HasValue) targetRow.Checked = setCheckedState.Value;
            }
            else
            {
                ListViewItem newRow = new ListViewItem();
                newRow.SubItems.Add(toolName);
                newRow.SubItems.Add(runningStatus);
                newRow.SubItems.Add(threadCount.ToString());
                if (setCheckedState.HasValue) newRow.Checked = setCheckedState.Value;

                _engineRows[toolName] = newRow;
                engineListView.Items.Add(newRow);
            }
        }

        private void button1_Click(object sender, EventArgs e)
        {
            BtnStartStop_Click(sender, e);
        }

        // Caps the event log at MaxStatusLogLines by dropping the oldest lines (FIFO) - a long continuous
        // run would otherwise grow StatusBox's text unbounded over hours/days.
        private const int MaxStatusLogLines = 5000;

        private void AppendStatusLine(string text)
        {
            StatusBox.AppendText(text);

            // .Lines splits on line breaks; every appended entry here ends in \r\n, so there's always a
            // trailing empty "line" after the last real one that shouldn't count toward the cap.
            string[] lines = StatusBox.Lines;
            int lineCount = lines.Length;
            if (lineCount > 0 && lines[lineCount - 1].Length == 0) lineCount--;

            if (lineCount > MaxStatusLogLines)
            {
                int firstKeptIndex = lineCount - MaxStatusLogLines;
                string[] trimmed = new string[lines.Length - firstKeptIndex];
                Array.Copy(lines, firstKeptIndex, trimmed, 0, trimmed.Length);

                StatusBox.Lines = trimmed;
                StatusBox.SelectionStart = StatusBox.Text.Length;
                StatusBox.ScrollToCaret();
            }
        }

        // Only the tool NAMES are captured from the checkbox selection - the actual ToolConfig objects are
        // re-resolved fresh from the repositories inside RunSelectedToolsOnceAsync every time it runs (see
        // below), so an edit to a tool/page/script yaml file made after checking a box still takes effect.
        private List<string> GetSelectedToolNames()
        {
            List<string> names = new List<string>();

            foreach (ListViewItem rowItem in engineListView.Items)
            {
                if (rowItem.Checked && rowItem.SubItems.Count > 1)
                {
                    names.Add(rowItem.SubItems[1].Text);
                }
            }

            return names;
        }

        // One full run of the selected tools to completion (or cancellation) - shared by the single manual
        // run button and the continuous run loop, so both go through the exact same execution path.
        private async Task RunSelectedToolsOnceAsync(List<string> toolNames, CancellationToken token)
        {
            // Reload tools/*.yaml, tools/pages/*.yaml and tools/scripts/*.yaml from disk fresh for every run
            // (not just once at app startup) - so a fixed destinationUrl, an edited script, a tweaked delay,
            // etc. takes effect on the very next run or continuous-loop cycle without restarting the app.
            ToolRepository.Instance.Reload();
            ScriptsRepository.Instance.Reload();

            List<ToolConfig> toolsToRun = new List<ToolConfig>();
            foreach (var name in toolNames)
            {
                var tool = ToolRepository.Instance.GetTool(name);
                if (tool != null)
                {
                    toolsToRun.Add(tool);
                }
                else
                {
                    Logger.Instance.Warn("SYSTEM", $"Selected tool '{name}' no longer exists after config reload - skipping it this run.");
                }
            }

            if (toolsToRun.Count == 0)
            {
                AppendStatusLine($"[{DateTime.Now:HH:mm:ss}] [SYSTEM] No selected tools resolved after config reload - nothing to run.\r\n");
                return;
            }

            ParallelScraperManager manager = new ParallelScraperManager(this, chkParallelPages.Checked);
            manager.DataScraped += Manager_DataScraped;

            try
            {
                AppendStatusLine($"[{DateTime.Now:HH:mm:ss}] [SYSTEM] Run initiated. Tools Parallel={chkParallelTools.Checked}, Pages Parallel={chkParallelPages.Checked}...\r\n");

                foreach (var t in toolsToRun)
                {
                    RegisterOrUpdateEngineState(t.ToolName, "Running...", t.ScrapPages.Count);
                }

                // 🔥 Toggle Execution Sequence Route based on chkParallelTools state configurations
                if (chkParallelTools.Checked)
                {
                    await manager.RunAllToolsInParallelAsync(toolsToRun, token);
                }
                else
                {
                    await manager.RunAllToolsInSequenceAsync(toolsToRun, token);
                }
            }
            finally
            {
                foreach (var t in toolsToRun)
                {
                    RegisterOrUpdateEngineState(t.ToolName, "Idle Ready", 0);
                }

                manager.DataScraped -= Manager_DataScraped;
            }
        }

        private async void BtnStartStop_Click(object sender, EventArgs e)
        {
            if (_cts != null)
            {
                AppendStatusLine($"[{DateTime.Now:HH:mm:ss}] [SYSTEM] Sending cancellation token request signals...\r\n");
                _cts.Cancel();
                return;
            }

            List<string> toolNames = GetSelectedToolNames();

            if (toolNames.Count == 0)
            {
                MessageBox.Show("Please select at least one automation engine checkbox row to execute.", "Control Monitor Info");
                return;
            }

            if (monitoringDashboard == null || monitoringDashboard.IsDisposed)
            {
                monitoringDashboard = new Form2();
                monitoringDashboard.Show();
            }

            _cts = new CancellationTokenSource();
            btnStartStop.Text = "STOP ENGINE EXECUTION";
            btnStartStop.BackColor = Color.Tomato;
            btnStartLoop.Enabled = false; // never run a manual run and the continuous loop at the same time - they'd share the same tools' CacheData/WebView2 profiles

            try
            {
                await RunSelectedToolsOnceAsync(toolNames, _cts.Token);
            }
            catch (OperationCanceledException)
            {
                AppendStatusLine($"[{DateTime.Now:HH:mm:ss}] [SYSTEM] Active operations successfully aborted by user.\r\n");
            }
            catch (Exception ex)
            {
                AppendStatusLine($"[{DateTime.Now:HH:mm:ss}] [ERROR] Execution fault: {ex.Message}\r\n");
            }
            finally
            {
                _cts.Dispose();
                _cts = null;
                btnStartStop.Text = "START SCRAPER ENGINE";
                btnStartStop.BackColor = Color.LightGreen;
                btnStartLoop.Enabled = true;
            }
        }

        // Repeats RunSelectedToolsOnceAsync on a fixed cadence (numIntervalSeconds), measured from the
        // START of one cycle to the START of the next - never runs a new cycle while the previous one is
        // still active. If a cycle takes longer than the interval, the next cycle starts immediately right
        // after it finishes (no overlap, no negative wait) instead of being skipped or stacked.
        private async void BtnStartLoop_Click(object sender, EventArgs e)
        {
            if (_loopCts != null)
            {
                AppendStatusLine($"[{DateTime.Now:HH:mm:ss}] [SYSTEM] Stopping continuous run loop (finishing current cycle first)...\r\n");
                _loopCts.Cancel();
                return;
            }

            List<string> toolNames = GetSelectedToolNames();

            if (toolNames.Count == 0)
            {
                MessageBox.Show("Please select at least one automation engine checkbox row to execute.", "Control Monitor Info");
                return;
            }

            if (monitoringDashboard == null || monitoringDashboard.IsDisposed)
            {
                monitoringDashboard = new Form2();
                monitoringDashboard.Show();
            }

            int intervalSeconds = (int)numIntervalSeconds.Value;

            _loopCts = new CancellationTokenSource();
            btnStartLoop.Text = "STOP CONTINUOUS RUN";
            btnStartLoop.BackColor = Color.Tomato;
            btnStartStop.Enabled = false;
            numIntervalSeconds.Enabled = false;
            engineListView.Enabled = false; // tool selection (which checkboxes are checked) is captured once at loop start; the underlying ToolConfig data itself is re-read from disk fresh every cycle via RunSelectedToolsOnceAsync's Reload() calls

            try
            {
                int cycle = 0;

                while (!_loopCts.Token.IsCancellationRequested)
                {
                    cycle++;
                    AppendStatusLine($"[{DateTime.Now:HH:mm:ss}] [SYSTEM] === Continuous run cycle #{cycle} starting (interval: {intervalSeconds}s) ===\r\n");

                    DateTime cycleStart = DateTime.Now;

                    try
                    {
                        await RunSelectedToolsOnceAsync(toolNames, _loopCts.Token);
                    }
                    catch (OperationCanceledException)
                    {
                        AppendStatusLine($"[{DateTime.Now:HH:mm:ss}] [SYSTEM] Continuous run loop stopped mid-cycle by user.\r\n");
                        break;
                    }
                    catch (Exception ex)
                    {
                        AppendStatusLine($"[{DateTime.Now:HH:mm:ss}] [ERROR] Cycle #{cycle} execution fault: {ex.Message}\r\n");
                    }

                    if (_loopCts.Token.IsCancellationRequested) break;

                    TimeSpan elapsed = DateTime.Now - cycleStart;
                    TimeSpan remaining = TimeSpan.FromSeconds(intervalSeconds) - elapsed;

                    if (remaining > TimeSpan.Zero)
                    {
                        AppendStatusLine($"[{DateTime.Now:HH:mm:ss}] [SYSTEM] Cycle #{cycle} finished in {elapsed.TotalSeconds:F1}s - waiting {remaining.TotalSeconds:F1}s until next run...\r\n");

                        try
                        {
                            await Task.Delay(remaining, _loopCts.Token);
                        }
                        catch (OperationCanceledException)
                        {
                            AppendStatusLine($"[{DateTime.Now:HH:mm:ss}] [SYSTEM] Continuous run loop stopped while waiting for the next cycle.\r\n");
                            break;
                        }
                    }
                    else
                    {
                        AppendStatusLine($"[{DateTime.Now:HH:mm:ss}] [SYSTEM] Cycle #{cycle} took {elapsed.TotalSeconds:F1}s, longer than the {intervalSeconds}s interval - starting the next cycle immediately (no overlap).\r\n");
                    }
                }
            }
            finally
            {
                _loopCts.Dispose();
                _loopCts = null;
                btnStartLoop.Text = "START CONTINUOUS RUN";
                btnStartLoop.BackColor = Color.LightGreen;
                btnStartStop.Enabled = true;
                numIntervalSeconds.Enabled = true;
                engineListView.Enabled = true;
            }
        }

        private void Manager_DataScraped(object sender, ScrapedDataEventArgs e)
        {
            if (this.InvokeRequired)
            {
                this.BeginInvoke((MethodInvoker)(() => Manager_DataScraped(sender, e)));
                return;
            }

            if (!string.IsNullOrEmpty(e.LogMessage))
            {
                AppendStatusLine($"[{e.Timestamp:HH:mm:ss}] [{e.ToolName}] {e.LogMessage}\r\n");
                return;
            }

            if (monitoringDashboard != null && !monitoringDashboard.IsDisposed && !string.IsNullOrEmpty(e.RawJson))
            {
                // 🛠️ STEP 1: Aggressively clean and unescape the raw JSON text from WebView2
                string cleanJson = e.RawJson.Trim();

                // Unescape the first layer of quotes if wrapped by WebView2
                if (cleanJson.StartsWith("\"") && cleanJson.EndsWith("\"") && cleanJson.Length > 1)
                {
                    cleanJson = cleanJson.Substring(1, cleanJson.Length - 2);
                    cleanJson = System.Text.RegularExpressions.Regex.Unescape(cleanJson);
                }

                // Handle double-serialized JSON strings (e.g. "\"[{\\\"name\\\":...}]\"")
                if (cleanJson.StartsWith("\"") && cleanJson.EndsWith("\"") && cleanJson.Length > 1)
                {
                    cleanJson = cleanJson.Substring(1, cleanJson.Length - 2);
                    cleanJson = System.Text.RegularExpressions.Regex.Unescape(cleanJson);
                }

                // Print a diagnostic message to track exactly what C# received
                AppendStatusLine($"[{DateTime.Now:HH:mm:ss}] [DEBUG UI] Processing type '{e.ReturnType}' | Sample payload: {(cleanJson.Length > 60 ? cleanJson.Substring(0, 60) : cleanJson)}...\r\n");

                List<StatusLineItem> translatedRows = null;

                try
                {
                    switch (e.ReturnType?.Trim())
                    {
                        case "ActiveProblems":
                            translatedRows = StatusLineDto.FromActiveProblems(e.ToolName, e.PageUrl, e.ScriptName, cleanJson);
                            break;

                        case "DeviceMeta":
                            translatedRows = StatusLineDto.FromDeviceMeta(e.ToolName, e.PageUrl, e.ScriptName, cleanJson);
                            break;

                        default:
                            AppendStatusLine($"[{DateTime.Now:HH:mm:ss}] [WARNING] Unhandled return type detected: '{e.ReturnType}'\r\n");
                            break;
                    }
                }
                catch (Exception ex)
                {
                    AppendStatusLine($"[{DateTime.Now:HH:mm:ss}] [ERROR UI] Deserialization crashed for type '{e.ReturnType}': {ex.Message}\r\n");
                }

                // 🛠️ STEP 2: Verify the DTO output before calling Form2
                if (translatedRows != null && translatedRows.Count > 0)
                {
                    AppendStatusLine($"[{DateTime.Now:HH:mm:ss}] [SUCCESS UI] Injecting {translatedRows.Count} rows into Form2.\r\n");
                    monitoringDashboard.ReplaceScriptResults(e.ToolName, e.PageUrl, e.ScriptName, translatedRows);
                }
                else if (translatedRows != null)
                {
                    // Still clear this script's previously-reported rows for this tool/page - a genuine
                    // 0-item result (e.g. all problems resolved) shouldn't leave stale rows behind. Note:
                    // this also clears on a transient/glitchy 0-item scrape, so a bad poll can temporarily
                    // blank a script's rows until the next successful run picks them back up.
                    AppendStatusLine($"[{DateTime.Now:HH:mm:ss}] [WARNING UI] DTO parsing completed, but returned 0 line items.\r\n");
                    monitoringDashboard.ReplaceScriptResults(e.ToolName, e.PageUrl, e.ScriptName, translatedRows);
                }
            }
        }
    }
}