diff --git a/DataProcessing/QuiverWallStreetBetsDataDownloader.cs b/DataProcessing/QuiverWallStreetBetsDataDownloader.cs
index 9451a70..1406b5d 100644
--- a/DataProcessing/QuiverWallStreetBetsDataDownloader.cs
+++ b/DataProcessing/QuiverWallStreetBetsDataDownloader.cs
@@ -123,25 +123,33 @@ public bool Run()
}
///
- /// Saves contents to disk
+ /// Merges contents with the stored file and saves them to disk, keeping one row per date or security and ticker
///
/// Final destination of the data
/// File name
/// Contents to write
- private void SaveContentToFile(string destinationFolder, string name, IEnumerable contents)
+ public static void SaveContentToFile(string destinationFolder, string name, IEnumerable contents)
{
var finalPath = Path.Combine(destinationFolder, $"{name.ToLowerInvariant()}.csv");
-
- var lines = new HashSet(contents);
- if (File.Exists(finalPath))
+
+ // One row per date (ticker file) or per security and ticker (universe file): a fetched value replaces the stored one.
+ // Two tickers can map to the same security on a date, so the ticker is part of the universe key.
+ // Older files can hold duplicate keys, written newest first in ticker files, so the first stored row wins
+ var isUniverse = destinationFolder.Contains("universe");
+ var keyColumns = isUniverse ? 2 : 1;
+ var linesByKey = new Dictionary();
+ var storedLines = File.Exists(finalPath) ? File.ReadAllLines(finalPath) : Array.Empty();
+ foreach (var line in storedLines.Where(x => !string.IsNullOrWhiteSpace(x)))
{
- foreach (var line in File.ReadAllLines(finalPath))
- {
- lines.Add(line);
- }
+ linesByKey.TryAdd(string.Join(',', line.Split(',').Take(keyColumns)), line);
+ }
+ foreach (var line in contents)
+ {
+ linesByKey[string.Join(',', line.Split(',').Take(keyColumns))] = line;
}
+ var lines = linesByKey.Values;
- var finalLines = destinationFolder.Contains("universe") ?
+ var finalLines = isUniverse ?
lines.OrderBy(x => x) :
lines.OrderBy(x => DateTime.ParseExact(x.Split(',').First(), "yyyyMMdd", CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal));
diff --git a/tests/QuiverWallStreetBetsDataDownloaderTests.cs b/tests/QuiverWallStreetBetsDataDownloaderTests.cs
new file mode 100644
index 0000000..584fbd1
--- /dev/null
+++ b/tests/QuiverWallStreetBetsDataDownloaderTests.cs
@@ -0,0 +1,156 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.IO;
+using NUnit.Framework;
+using NUnit.Framework.Legacy;
+using QuantConnect.DataProcessing;
+
+namespace QuantConnect.DataLibrary.Tests
+{
+ [TestFixture]
+ public class QuiverWallStreetBetsDataDownloaderTests
+ {
+ private string _root;
+ private string _universe;
+
+ [SetUp]
+ public void SetUp()
+ {
+ _root = Path.Combine(Path.GetTempPath(), "quiver-wsb-tests-" + Guid.NewGuid().ToString("N"));
+ _universe = Path.Combine(_root, "universe");
+ Directory.CreateDirectory(_universe);
+ }
+
+ [TearDown]
+ public void TearDown()
+ {
+ Directory.Delete(_root, true);
+ }
+
+ [Test]
+ public void RevisedValueReplacesTheStoredRowOfTheSameDate()
+ {
+ File.WriteAllLines(Path.Combine(_root, "abc.csv"), new[]
+ {
+ "20260807,5,3,0.2",
+ "20260808,10,1,0.1"
+ });
+
+ QuiverWallStreetBetsDataDownloader.SaveContentToFile(_root, "ABC", new[]
+ {
+ "20260808,12,1,0.1",
+ "20260809,7,2,-0.3"
+ });
+
+ CollectionAssert.AreEqual(new[]
+ {
+ "20260807,5,3,0.2",
+ "20260808,12,1,0.1",
+ "20260809,7,2,-0.3"
+ }, File.ReadAllLines(Path.Combine(_root, "abc.csv")));
+ }
+
+ [Test]
+ public void RevisedValueReplacesTheStoredUniverseRowOfTheSameSecurity()
+ {
+ File.WriteAllLines(Path.Combine(_universe, "20260808.csv"), new[]
+ {
+ "AAPL R735QTJ8XC9X,AAPL,10,1,0.1",
+ "GME SC72NCBXXAHX,GME,50,2,0.5"
+ });
+
+ QuiverWallStreetBetsDataDownloader.SaveContentToFile(_universe, "20260808", new[]
+ {
+ "AAPL R735QTJ8XC9X,AAPL,12,1,0.1"
+ });
+
+ CollectionAssert.AreEqual(new[]
+ {
+ "AAPL R735QTJ8XC9X,AAPL,12,1,0.1",
+ "GME SC72NCBXXAHX,GME,50,2,0.5"
+ }, File.ReadAllLines(Path.Combine(_universe, "20260808.csv")));
+ }
+
+ [Test]
+ public void TickersSharingASecurityKeepTheirOwnUniverseRows()
+ {
+ File.WriteAllLines(Path.Combine(_universe, "20250210.csv"), new[]
+ {
+ "ABX R735QTJ8XC9X,B,9,50,0.1",
+ "ABX R735QTJ8XC9X,GOLD,4,108,0.64043"
+ });
+
+ QuiverWallStreetBetsDataDownloader.SaveContentToFile(_universe, "20250210", new[]
+ {
+ "ABX R735QTJ8XC9X,B,10,49,0.11145",
+ "ABX R735QTJ8XC9X,GOLD,4,108,0.64043"
+ });
+
+ CollectionAssert.AreEqual(new[]
+ {
+ "ABX R735QTJ8XC9X,B,10,49,0.11145",
+ "ABX R735QTJ8XC9X,GOLD,4,108,0.64043"
+ }, File.ReadAllLines(Path.Combine(_universe, "20250210.csv")));
+ }
+
+ // The previous merge put the fetched rows before the stored ones and sorted by date with a stable sort,
+ // so a stored duplicate pair has the newest value first
+ [Test]
+ public void DuplicatesAlreadyStoredKeepTheNewestRow()
+ {
+ File.WriteAllLines(Path.Combine(_root, "abc.csv"), new[]
+ {
+ "20260808,12,1,0.1",
+ "20260808,10,1,0.1"
+ });
+
+ QuiverWallStreetBetsDataDownloader.SaveContentToFile(_root, "ABC", new[]
+ {
+ "20260809,7,2,-0.3"
+ });
+
+ CollectionAssert.AreEqual(new[]
+ {
+ "20260808,12,1,0.1",
+ "20260809,7,2,-0.3"
+ }, File.ReadAllLines(Path.Combine(_root, "abc.csv")));
+ }
+
+ [Test]
+ public void DuplicatesAlreadyStoredAreReplacedByTheFetchedRow()
+ {
+ File.WriteAllLines(Path.Combine(_universe, "20260808.csv"), new[]
+ {
+ "AAPL R735QTJ8XC9X,AAPL,10,1,0.1",
+ "AAPL R735QTJ8XC9X,AAPL,12,1,0.1",
+ "GME SC72NCBXXAHX,GME,50,2,0.5"
+ });
+
+ QuiverWallStreetBetsDataDownloader.SaveContentToFile(_universe, "20260808", new[]
+ {
+ "AAPL R735QTJ8XC9X,AAPL,13,1,0.1"
+ });
+
+ CollectionAssert.AreEqual(new[]
+ {
+ "AAPL R735QTJ8XC9X,AAPL,13,1,0.1",
+ "GME SC72NCBXXAHX,GME,50,2,0.5"
+ }, File.ReadAllLines(Path.Combine(_universe, "20260808.csv")));
+ }
+ }
+}