diff --git a/DataProcessing/QuiverWallStreetBetsDataDownloader.cs b/DataProcessing/QuiverWallStreetBetsDataDownloader.cs index 9451a70..1406b5d 100644 --- a/DataProcessing/QuiverWallStreetBetsDataDownloader.cs +++ b/DataProcessing/QuiverWallStreetBetsDataDownloader.cs @@ -123,25 +123,33 @@ public bool Run() } /// - /// Saves contents to disk + /// Merges contents with the stored file and saves them to disk, keeping one row per date or security and ticker /// /// Final destination of the data /// File name /// Contents to write - private void SaveContentToFile(string destinationFolder, string name, IEnumerable contents) + public static void SaveContentToFile(string destinationFolder, string name, IEnumerable contents) { var finalPath = Path.Combine(destinationFolder, $"{name.ToLowerInvariant()}.csv"); - - var lines = new HashSet(contents); - if (File.Exists(finalPath)) + + // One row per date (ticker file) or per security and ticker (universe file): a fetched value replaces the stored one. + // Two tickers can map to the same security on a date, so the ticker is part of the universe key. + // Older files can hold duplicate keys, written newest first in ticker files, so the first stored row wins + var isUniverse = destinationFolder.Contains("universe"); + var keyColumns = isUniverse ? 2 : 1; + var linesByKey = new Dictionary(); + var storedLines = File.Exists(finalPath) ? File.ReadAllLines(finalPath) : Array.Empty(); + foreach (var line in storedLines.Where(x => !string.IsNullOrWhiteSpace(x))) { - foreach (var line in File.ReadAllLines(finalPath)) - { - lines.Add(line); - } + linesByKey.TryAdd(string.Join(',', line.Split(',').Take(keyColumns)), line); + } + foreach (var line in contents) + { + linesByKey[string.Join(',', line.Split(',').Take(keyColumns))] = line; } + var lines = linesByKey.Values; - var finalLines = destinationFolder.Contains("universe") ? + var finalLines = isUniverse ? lines.OrderBy(x => x) : lines.OrderBy(x => DateTime.ParseExact(x.Split(',').First(), "yyyyMMdd", CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal)); diff --git a/tests/QuiverWallStreetBetsDataDownloaderTests.cs b/tests/QuiverWallStreetBetsDataDownloaderTests.cs new file mode 100644 index 0000000..584fbd1 --- /dev/null +++ b/tests/QuiverWallStreetBetsDataDownloaderTests.cs @@ -0,0 +1,156 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using System; +using System.IO; +using NUnit.Framework; +using NUnit.Framework.Legacy; +using QuantConnect.DataProcessing; + +namespace QuantConnect.DataLibrary.Tests +{ + [TestFixture] + public class QuiverWallStreetBetsDataDownloaderTests + { + private string _root; + private string _universe; + + [SetUp] + public void SetUp() + { + _root = Path.Combine(Path.GetTempPath(), "quiver-wsb-tests-" + Guid.NewGuid().ToString("N")); + _universe = Path.Combine(_root, "universe"); + Directory.CreateDirectory(_universe); + } + + [TearDown] + public void TearDown() + { + Directory.Delete(_root, true); + } + + [Test] + public void RevisedValueReplacesTheStoredRowOfTheSameDate() + { + File.WriteAllLines(Path.Combine(_root, "abc.csv"), new[] + { + "20260807,5,3,0.2", + "20260808,10,1,0.1" + }); + + QuiverWallStreetBetsDataDownloader.SaveContentToFile(_root, "ABC", new[] + { + "20260808,12,1,0.1", + "20260809,7,2,-0.3" + }); + + CollectionAssert.AreEqual(new[] + { + "20260807,5,3,0.2", + "20260808,12,1,0.1", + "20260809,7,2,-0.3" + }, File.ReadAllLines(Path.Combine(_root, "abc.csv"))); + } + + [Test] + public void RevisedValueReplacesTheStoredUniverseRowOfTheSameSecurity() + { + File.WriteAllLines(Path.Combine(_universe, "20260808.csv"), new[] + { + "AAPL R735QTJ8XC9X,AAPL,10,1,0.1", + "GME SC72NCBXXAHX,GME,50,2,0.5" + }); + + QuiverWallStreetBetsDataDownloader.SaveContentToFile(_universe, "20260808", new[] + { + "AAPL R735QTJ8XC9X,AAPL,12,1,0.1" + }); + + CollectionAssert.AreEqual(new[] + { + "AAPL R735QTJ8XC9X,AAPL,12,1,0.1", + "GME SC72NCBXXAHX,GME,50,2,0.5" + }, File.ReadAllLines(Path.Combine(_universe, "20260808.csv"))); + } + + [Test] + public void TickersSharingASecurityKeepTheirOwnUniverseRows() + { + File.WriteAllLines(Path.Combine(_universe, "20250210.csv"), new[] + { + "ABX R735QTJ8XC9X,B,9,50,0.1", + "ABX R735QTJ8XC9X,GOLD,4,108,0.64043" + }); + + QuiverWallStreetBetsDataDownloader.SaveContentToFile(_universe, "20250210", new[] + { + "ABX R735QTJ8XC9X,B,10,49,0.11145", + "ABX R735QTJ8XC9X,GOLD,4,108,0.64043" + }); + + CollectionAssert.AreEqual(new[] + { + "ABX R735QTJ8XC9X,B,10,49,0.11145", + "ABX R735QTJ8XC9X,GOLD,4,108,0.64043" + }, File.ReadAllLines(Path.Combine(_universe, "20250210.csv"))); + } + + // The previous merge put the fetched rows before the stored ones and sorted by date with a stable sort, + // so a stored duplicate pair has the newest value first + [Test] + public void DuplicatesAlreadyStoredKeepTheNewestRow() + { + File.WriteAllLines(Path.Combine(_root, "abc.csv"), new[] + { + "20260808,12,1,0.1", + "20260808,10,1,0.1" + }); + + QuiverWallStreetBetsDataDownloader.SaveContentToFile(_root, "ABC", new[] + { + "20260809,7,2,-0.3" + }); + + CollectionAssert.AreEqual(new[] + { + "20260808,12,1,0.1", + "20260809,7,2,-0.3" + }, File.ReadAllLines(Path.Combine(_root, "abc.csv"))); + } + + [Test] + public void DuplicatesAlreadyStoredAreReplacedByTheFetchedRow() + { + File.WriteAllLines(Path.Combine(_universe, "20260808.csv"), new[] + { + "AAPL R735QTJ8XC9X,AAPL,10,1,0.1", + "AAPL R735QTJ8XC9X,AAPL,12,1,0.1", + "GME SC72NCBXXAHX,GME,50,2,0.5" + }); + + QuiverWallStreetBetsDataDownloader.SaveContentToFile(_universe, "20260808", new[] + { + "AAPL R735QTJ8XC9X,AAPL,13,1,0.1" + }); + + CollectionAssert.AreEqual(new[] + { + "AAPL R735QTJ8XC9X,AAPL,13,1,0.1", + "GME SC72NCBXXAHX,GME,50,2,0.5" + }, File.ReadAllLines(Path.Combine(_universe, "20260808.csv"))); + } + } +}