From c31ddd29f470c0c2778ced12edc62d9421420ab2 Mon Sep 17 00:00:00 2001 From: "quantconnect-kronos[bot]" <346399058+quantconnect-kronos[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 23:13:52 +0000 Subject: [PATCH 1/2] Keep one row per date or security in WallStreetBets files Merging by whole line kept a revised value next to the stored one for the same date or security. Merge by the row key instead, letting fetched rows replace stored ones. Co-Authored-By: Claude Opus 5.5 --- .../QuiverWallStreetBetsDataDownloader.cs | 23 +-- ...QuiverWallStreetBetsDataDownloaderTests.cs | 134 ++++++++++++++++++ 2 files changed, 148 insertions(+), 9 deletions(-) create mode 100644 tests/QuiverWallStreetBetsDataDownloaderTests.cs diff --git a/DataProcessing/QuiverWallStreetBetsDataDownloader.cs b/DataProcessing/QuiverWallStreetBetsDataDownloader.cs index 9451a70..48b741c 100644 --- a/DataProcessing/QuiverWallStreetBetsDataDownloader.cs +++ b/DataProcessing/QuiverWallStreetBetsDataDownloader.cs @@ -123,23 +123,28 @@ public bool Run() } /// - /// Saves contents to disk + /// Merges contents with the stored file and saves them to disk, keeping one row per date or security /// /// Final destination of the data /// File name /// Contents to write - private void SaveContentToFile(string destinationFolder, string name, IEnumerable contents) + public static void SaveContentToFile(string destinationFolder, string name, IEnumerable contents) { var finalPath = Path.Combine(destinationFolder, $"{name.ToLowerInvariant()}.csv"); - - var lines = new HashSet(contents); - if (File.Exists(finalPath)) + + // One row per date (ticker file) or per security (universe file): a fetched value replaces the stored one. + // Older files can hold duplicate keys, written newest first in ticker files, so the first stored row wins + var linesByKey = new Dictionary(); + var storedLines = File.Exists(finalPath) ? File.ReadAllLines(finalPath) : Array.Empty(); + foreach (var line in storedLines.Where(x => !string.IsNullOrWhiteSpace(x))) { - foreach (var line in File.ReadAllLines(finalPath)) - { - lines.Add(line); - } + linesByKey.TryAdd(line.Split(',')[0], line); + } + foreach (var line in contents) + { + linesByKey[line.Split(',')[0]] = line; } + var lines = linesByKey.Values; var finalLines = destinationFolder.Contains("universe") ? lines.OrderBy(x => x) : diff --git a/tests/QuiverWallStreetBetsDataDownloaderTests.cs b/tests/QuiverWallStreetBetsDataDownloaderTests.cs new file mode 100644 index 0000000..9ffbe3c --- /dev/null +++ b/tests/QuiverWallStreetBetsDataDownloaderTests.cs @@ -0,0 +1,134 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using System; +using System.IO; +using NUnit.Framework; +using NUnit.Framework.Legacy; +using QuantConnect.DataProcessing; + +namespace QuantConnect.DataLibrary.Tests +{ + [TestFixture] + public class QuiverWallStreetBetsDataDownloaderTests + { + private string _root; + private string _universe; + + [SetUp] + public void SetUp() + { + _root = Path.Combine(Path.GetTempPath(), "quiver-wsb-tests-" + Guid.NewGuid().ToString("N")); + _universe = Path.Combine(_root, "universe"); + Directory.CreateDirectory(_universe); + } + + [TearDown] + public void TearDown() + { + Directory.Delete(_root, true); + } + + [Test] + public void RevisedValueReplacesTheStoredRowOfTheSameDate() + { + File.WriteAllLines(Path.Combine(_root, "abc.csv"), new[] + { + "20260807,5,3,0.2", + "20260808,10,1,0.1" + }); + + QuiverWallStreetBetsDataDownloader.SaveContentToFile(_root, "ABC", new[] + { + "20260808,12,1,0.1", + "20260809,7,2,-0.3" + }); + + CollectionAssert.AreEqual(new[] + { + "20260807,5,3,0.2", + "20260808,12,1,0.1", + "20260809,7,2,-0.3" + }, File.ReadAllLines(Path.Combine(_root, "abc.csv"))); + } + + [Test] + public void RevisedValueReplacesTheStoredUniverseRowOfTheSameSecurity() + { + File.WriteAllLines(Path.Combine(_universe, "20260808.csv"), new[] + { + "AAPL R735QTJ8XC9X,AAPL,10,1,0.1", + "GME SC72NCBXXAHX,GME,50,2,0.5" + }); + + QuiverWallStreetBetsDataDownloader.SaveContentToFile(_universe, "20260808", new[] + { + "AAPL R735QTJ8XC9X,AAPL,12,1,0.1" + }); + + CollectionAssert.AreEqual(new[] + { + "AAPL R735QTJ8XC9X,AAPL,12,1,0.1", + "GME SC72NCBXXAHX,GME,50,2,0.5" + }, File.ReadAllLines(Path.Combine(_universe, "20260808.csv"))); + } + + // The previous merge put the fetched rows before the stored ones and sorted by date with a stable sort, + // so a stored duplicate pair has the newest value first + [Test] + public void DuplicatesAlreadyStoredKeepTheNewestRow() + { + File.WriteAllLines(Path.Combine(_root, "abc.csv"), new[] + { + "20260808,12,1,0.1", + "20260808,10,1,0.1" + }); + + QuiverWallStreetBetsDataDownloader.SaveContentToFile(_root, "ABC", new[] + { + "20260809,7,2,-0.3" + }); + + CollectionAssert.AreEqual(new[] + { + "20260808,12,1,0.1", + "20260809,7,2,-0.3" + }, File.ReadAllLines(Path.Combine(_root, "abc.csv"))); + } + + [Test] + public void DuplicatesAlreadyStoredAreReplacedByTheFetchedRow() + { + File.WriteAllLines(Path.Combine(_universe, "20260808.csv"), new[] + { + "AAPL R735QTJ8XC9X,AAPL,10,1,0.1", + "AAPL R735QTJ8XC9X,AAPL,12,1,0.1", + "GME SC72NCBXXAHX,GME,50,2,0.5" + }); + + QuiverWallStreetBetsDataDownloader.SaveContentToFile(_universe, "20260808", new[] + { + "AAPL R735QTJ8XC9X,AAPL,13,1,0.1" + }); + + CollectionAssert.AreEqual(new[] + { + "AAPL R735QTJ8XC9X,AAPL,13,1,0.1", + "GME SC72NCBXXAHX,GME,50,2,0.5" + }, File.ReadAllLines(Path.Combine(_universe, "20260808.csv"))); + } + } +} From 367d21d6ca110a2e6f631a21626706d1ecc07ae8 Mon Sep 17 00:00:00 2001 From: "quantconnect-kronos[bot]" <346399058+quantconnect-kronos[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 23:30:46 +0000 Subject: [PATCH 2/2] Key universe rows by security and ticker Two tickers can map to the same security identifier on a date, so keying universe rows by the identifier alone dropped one of them. Co-Authored-By: Claude Opus 5.5 --- .../QuiverWallStreetBetsDataDownloader.cs | 13 ++++++----- ...QuiverWallStreetBetsDataDownloaderTests.cs | 22 +++++++++++++++++++ 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/DataProcessing/QuiverWallStreetBetsDataDownloader.cs b/DataProcessing/QuiverWallStreetBetsDataDownloader.cs index 48b741c..1406b5d 100644 --- a/DataProcessing/QuiverWallStreetBetsDataDownloader.cs +++ b/DataProcessing/QuiverWallStreetBetsDataDownloader.cs @@ -123,7 +123,7 @@ public bool Run() } /// - /// Merges contents with the stored file and saves them to disk, keeping one row per date or security + /// Merges contents with the stored file and saves them to disk, keeping one row per date or security and ticker /// /// Final destination of the data /// File name @@ -132,21 +132,24 @@ public static void SaveContentToFile(string destinationFolder, string name, IEnu { var finalPath = Path.Combine(destinationFolder, $"{name.ToLowerInvariant()}.csv"); - // One row per date (ticker file) or per security (universe file): a fetched value replaces the stored one. + // One row per date (ticker file) or per security and ticker (universe file): a fetched value replaces the stored one. + // Two tickers can map to the same security on a date, so the ticker is part of the universe key. // Older files can hold duplicate keys, written newest first in ticker files, so the first stored row wins + var isUniverse = destinationFolder.Contains("universe"); + var keyColumns = isUniverse ? 2 : 1; var linesByKey = new Dictionary(); var storedLines = File.Exists(finalPath) ? File.ReadAllLines(finalPath) : Array.Empty(); foreach (var line in storedLines.Where(x => !string.IsNullOrWhiteSpace(x))) { - linesByKey.TryAdd(line.Split(',')[0], line); + linesByKey.TryAdd(string.Join(',', line.Split(',').Take(keyColumns)), line); } foreach (var line in contents) { - linesByKey[line.Split(',')[0]] = line; + linesByKey[string.Join(',', line.Split(',').Take(keyColumns))] = line; } var lines = linesByKey.Values; - var finalLines = destinationFolder.Contains("universe") ? + var finalLines = isUniverse ? lines.OrderBy(x => x) : lines.OrderBy(x => DateTime.ParseExact(x.Split(',').First(), "yyyyMMdd", CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal)); diff --git a/tests/QuiverWallStreetBetsDataDownloaderTests.cs b/tests/QuiverWallStreetBetsDataDownloaderTests.cs index 9ffbe3c..584fbd1 100644 --- a/tests/QuiverWallStreetBetsDataDownloaderTests.cs +++ b/tests/QuiverWallStreetBetsDataDownloaderTests.cs @@ -86,6 +86,28 @@ public void RevisedValueReplacesTheStoredUniverseRowOfTheSameSecurity() }, File.ReadAllLines(Path.Combine(_universe, "20260808.csv"))); } + [Test] + public void TickersSharingASecurityKeepTheirOwnUniverseRows() + { + File.WriteAllLines(Path.Combine(_universe, "20250210.csv"), new[] + { + "ABX R735QTJ8XC9X,B,9,50,0.1", + "ABX R735QTJ8XC9X,GOLD,4,108,0.64043" + }); + + QuiverWallStreetBetsDataDownloader.SaveContentToFile(_universe, "20250210", new[] + { + "ABX R735QTJ8XC9X,B,10,49,0.11145", + "ABX R735QTJ8XC9X,GOLD,4,108,0.64043" + }); + + CollectionAssert.AreEqual(new[] + { + "ABX R735QTJ8XC9X,B,10,49,0.11145", + "ABX R735QTJ8XC9X,GOLD,4,108,0.64043" + }, File.ReadAllLines(Path.Combine(_universe, "20250210.csv"))); + } + // The previous merge put the fetched rows before the stored ones and sorted by date with a stable sort, // so a stored duplicate pair has the newest value first [Test]