fix map download and scynchronizaiton

AutoRegistrator/AutoRegistrator.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Data.Entity;
 using System.Diagnostics;
 using System.IO;
 using System.Linq;
@@ -153,31 +154,66 @@ private void OnRapidChanged()
 
         private void SynchronizeMapsFromSpringFiles()
         {
-            if (GlobalConst.Mode == ModeType.Live)
-            {
-                var db = new ZkDataContext();
+            if (GlobalConst.Mode != ModeType.Live) return;
 
-                var processedFiles = db.ResourceContentFiles.Where(x => x.Resource.TypeID == ResourceType.Map).Select(x => x.FileName.ToLower()).Distinct().ToLookup(x => x);
-                var triedFiles = db.SpringFilesUnitsyncAttempts.Select(x => x.FileName.ToLower()).Distinct().ToLookup(x => x);
+            HashSet<string> processedFiles, triedFiles;
+            using (var db = new ZkDataContext())
+            {
+                processedFiles = new HashSet<string>(
+                    db.ResourceContentFiles.AsNoTracking()
+                        .Where(x => x.Resource.TypeID == ResourceType.Map)
+                        .Select(x => x.FileName.ToLower()));
+                triedFiles = new HashSet<string>(
+                    db.SpringFilesUnitsyncAttempts.AsNoTracking().Select(x => x.FileName.ToLower()));
+            }
 
-                var webSyncer = new WebFolderSyncer();
+            var webSyncer = new WebFolderSyncer();
+            var autoregMaps = Path.Combine(Paths.WritableDirectory, "maps");
+            var contentMaps = Path.Combine(sitePath, "content", "maps");
+            if (!Directory.Exists(contentMaps)) Directory.CreateDirectory(contentMaps);
 
-                foreach (var file in webSyncer.GetFileList())
+            foreach (var file in webSyncer.GetFileList())
+            {
+                var fileLc = file.ToLower();
+                var alreadyRegistered = processedFiles.Contains(fileLc);
+                var alreadyAttempted = triedFiles.Contains(fileLc);
+                var contentFile = Path.Combine(contentMaps, file);
+                var hasLocal = File.Exists(contentFile);
+
+                // skip cases:
+                //  - registered AND local copy present → nothing to do
+                //  - previously attempted but never made it into DB → known failure, don't retry
+                if (alreadyRegistered && hasLocal) continue;
+                if (alreadyAttempted && !alreadyRegistered) continue;
+
+                if (!webSyncer.DownloadFile(autoregMaps, file)) continue;
+                var srcFile = Path.Combine(autoregMaps, file);
+
+                var registered = alreadyRegistered;
+                if (!alreadyRegistered)
                 {
-                    if (processedFiles.Contains(file.ToLower()) || triedFiles.Contains(file.ToLower())) continue;
-
-                    webSyncer.DownloadFile(Path.Combine(Paths.WritableDirectory,"maps"), file);
-
-                    UnitSyncer.Scan();
+                    var results = UnitSyncer.Scan();
+                    var ourResult = results?.FirstOrDefault(r => string.Equals(r.ResourceInfo?.ArchiveName, file, StringComparison.OrdinalIgnoreCase));
+                    registered = ourResult != null && ourResult.Status != UnitSyncer.ResourceFileStatus.RegistrationError;
 
-                    db.SpringFilesUnitsyncAttempts.Add(new SpringFilesUnitsyncAttempt() { FileName = file });
-                    db.SaveChanges();
+                    using (var db = new ZkDataContext())
+                    {
+                        db.SpringFilesUnitsyncAttempts.Add(new SpringFilesUnitsyncAttempt() { FileName = file });
+                        db.SaveChanges();
+                    }
+                    triedFiles.Add(fileLc);
+                }
 
-                    try
+                if (registered && !hasLocal)
+                {
+                    try { File.Copy(srcFile, contentFile); }
+                    catch (Exception ex)
                     {
-                        File.Delete(Path.Combine(Paths.WritableDirectory, "maps", file));
-                    } catch (Exception ex) { }
+                        Trace.TraceWarning("Copy {0} to content/maps failed: {1}", file, ex.Message);
+                    }
                 }
+
+                try { File.Delete(srcFile); } catch { }
             }
         }
 

AutoRegistrator/WebFolderSync.cs

@@ -1,69 +1,62 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
-using System.Linq;
 using System.Net;
-using System.Text.RegularExpressions;
+using Newtonsoft.Json;
+using ZkData;
 
 namespace ZeroKWeb
 {
     public class WebFolderSyncer
     {
-        private string urlSource;
-
-        public WebFolderSyncer(string urlSource = "http://api.springfiles.com/files/maps/")
-        {
-            this.urlSource = urlSource;
-        }
-
         public List<string> GetFileList()
         {
+            var ret = new List<string>();
             using (var wc = new WebClient())
             {
-                var str = wc.DownloadString(urlSource);
-                return (from Match m in Regex.Matches(str, "<a href=\"([^\"]+)\">([^<]+)</a>")
-                        where m.Success && m.Groups[1].Value == m.Groups[2].Value
-                        select m.Groups[1].Value).ToList();
+                var url = $"{GlobalConst.SpringfilesBaseUrl}json.php?category=map&springname=*&logical=or&nosensitive=on&limit=1000000&offset=0";
+                var json = wc.DownloadString(url);
+                var entries = JsonConvert.DeserializeObject<List<SpringfilesEntry>>(json);
+                if (entries == null) return ret;
+                foreach (var e in entries)
+                {
+                    if (e?.path == "maps" && !string.IsNullOrEmpty(e.filename) && e.mirrors != null && e.mirrors.Length > 0)
+                        ret.Add(e.filename);
+                }
             }
+            return ret;
         }
 
-        public void DownloadFile(string targetFolder, string file)
+        public bool DownloadFile(string targetFolder, string file)
         {
             if (!Directory.Exists(targetFolder)) Directory.CreateDirectory(targetFolder);
             var targetFile = Path.Combine(targetFolder, file);
-            if (!File.Exists(targetFile))
+            if (File.Exists(targetFile)) return true;
+
+            var tempFile = Path.GetTempFileName();
+            using (var wc = new WebClient())
             {
-                using (var wc = new WebClient())
+                try
                 {
-                    var tempFile = Path.GetTempFileName();
-                    try
-                    {
-                        wc.DownloadFile(urlSource + file, tempFile);
-                    }
-                    catch (Exception ex)
-                    {
-                        Trace.TraceWarning("Download of file {0} failed: {1}", file, ex.Message);
-                        File.Delete(tempFile);
-                    }
-                    try
-                    {
-                        File.Move(tempFile, targetFile);
-                    }
-                    catch { }
+                    wc.DownloadFile($"{GlobalConst.SpringfilesBaseUrl}files/maps/{file}", tempFile);
+                    File.Move(tempFile, targetFile);
+                    return true;
+                }
+                catch (Exception ex)
+                {
+                    Trace.TraceWarning("Springfiles download of {0} failed: {1}", file, ex.Message);
+                    try { File.Delete(tempFile); } catch { }
+                    return false;
                 }
             }
         }
 
-
-        public void SynchronizeFolders(string targetFolder)
+        private class SpringfilesEntry
         {
-            if (!Directory.Exists(targetFolder)) Directory.CreateDirectory(targetFolder);
-            foreach (var file in GetFileList())
-            {
-                DownloadFile(targetFolder, file);
-            }
+            public string filename { get; set; }
+            public string path { get; set; }
+            public string[] mirrors { get; set; }
         }
-
     }
 }

Shared/PlasmaShared/GlobalConst.cs

@@ -79,8 +79,6 @@ static void SetMode(ModeType newMode)
                     break;
             }
 
-            if (IsLongAfterSteam) DefaultDownloadMirrors = new[] { BaseSiteUrl +"/content/%t/%f" };
-
             ResourceBaseUrl = string.Format("{0}/Resources", BaseSiteUrl);
             BaseImageUrl = string.Format("{0}/img/", BaseSiteUrl);
             SelfUpdaterBaseUrl = string.Format("{0}/lobby", BaseSiteUrl);
@@ -98,6 +96,8 @@ static void SetMode(ModeType newMode)
         public static string BaseImageUrl;
         public static string BaseSiteUrl;
 
+        public const string SpringfilesBaseUrl = "https://springfiles.springrts.com/";
+
         public static string DefaultZkTag => Mode == ModeType.Live ? "zk:stable" : "zk:test";
         public static string DefaultChobbyTag => Mode == ModeType.Live ? "zkmenu:stable" : "zkmenu:test";
 
@@ -235,7 +235,6 @@ static void SetMode(ModeType newMode)
 
         public static string ResourceBaseUrl;
         public static string SelfUpdaterBaseUrl;
-        public static string[] DefaultDownloadMirrors = {};
         public static string LobbyServerHost;
         public static int LobbyServerPort;
         public static bool LobbyServerUpdateSpectatorsInstantly = false;