Import extra region folders into slime format

api/src/main/java/com/infernalsuite/asp/api/AdvancedSlimePaperAPI.java

@@ -8,6 +8,7 @@
 import com.infernalsuite.asp.api.exceptions.WorldLoadedException;
 import com.infernalsuite.asp.api.exceptions.WorldTooBigException;
 import com.infernalsuite.asp.api.loaders.SlimeSerializationAdapter;
+import com.infernalsuite.asp.api.world.ExtraRegionFolder;
 import com.infernalsuite.asp.api.world.SlimeWorld;
 import com.infernalsuite.asp.api.world.SlimeWorldInstance;
 import com.infernalsuite.asp.api.world.properties.SlimePropertyMap;
@@ -132,7 +133,11 @@ SlimeWorld readWorld(SlimeLoader loader, String worldName, boolean readOnly, Sli
      * @throws WorldTooBigException        if the world is too big to be imported into the SRF.
      * @throws IOException                 if the world could not be read or stored.
      */
-    SlimeWorld readVanillaWorld(File worldDir, String worldName, @Nullable SlimeLoader loader) throws InvalidWorldException, WorldLoadedException, WorldTooBigException, IOException, WorldAlreadyExistsException;
+    default SlimeWorld readVanillaWorld(File worldDir, String worldName, @Nullable SlimeLoader loader) throws InvalidWorldException, WorldLoadedException, WorldTooBigException, IOException, WorldAlreadyExistsException {
+        return readVanillaWorld(worldDir, worldName, loader, List.of());
+    }
+
+    SlimeWorld readVanillaWorld(File worldDir, String worldName, @Nullable SlimeLoader loader, List<ExtraRegionFolder> extraRegionFolders) throws InvalidWorldException, WorldLoadedException, WorldTooBigException, IOException, WorldAlreadyExistsException;
 
     /**
      * Returns the {@link SlimeSerializationAdapter} used to serialize and deserialize SlimeWorlds.

api/src/main/java/com/infernalsuite/asp/api/world/ExtraRegionFolder.java

@@ -0,0 +1,12 @@
+package com.infernalsuite.asp.api.world;
+
+public record ExtraRegionFolder(String directory, String extraDataKey) {
+
+    public static ExtraRegionFolder parse(String value) {
+        int separator = value.indexOf('=');
+        if (separator < 0) {
+            return new ExtraRegionFolder(value, value);
+        }
+        return new ExtraRegionFolder(value.substring(0, separator), value.substring(separator + 1));
+    }
+}

aspaper-server/src/main/java/com/infernalsuite/asp/AdvancedSlimePaper.java

@@ -6,6 +6,7 @@
 import com.infernalsuite.asp.api.exceptions.*;
 import com.infernalsuite.asp.api.loaders.SlimeLoader;
 import com.infernalsuite.asp.api.loaders.SlimeSerializationAdapter;
+import com.infernalsuite.asp.api.world.ExtraRegionFolder;
 import com.infernalsuite.asp.api.world.SlimeWorld;
 import com.infernalsuite.asp.api.world.SlimeWorldInstance;
 import com.infernalsuite.asp.api.world.properties.SlimePropertyMap;
@@ -190,9 +191,10 @@ public void migrateWorld(String worldName, SlimeLoader currentLoader, SlimeLoade
     }
 
     @Override
-    public SlimeWorld readVanillaWorld(File worldDir, String worldName, SlimeLoader loader) throws InvalidWorldException, WorldLoadedException, WorldTooBigException, IOException, WorldAlreadyExistsException {
+    public SlimeWorld readVanillaWorld(File worldDir, String worldName, SlimeLoader loader, List<ExtraRegionFolder> extraRegionFolders) throws InvalidWorldException, WorldLoadedException, WorldTooBigException, IOException, WorldAlreadyExistsException {
         Objects.requireNonNull(worldDir, "World directory cannot be null");
         Objects.requireNonNull(worldName, "World name cannot be null");
+        Objects.requireNonNull(extraRegionFolders, "Extra region folders cannot be null");
 
         if (loader != null && loader.worldExists(worldName)) {
             throw new WorldAlreadyExistsException(worldName);
@@ -207,7 +209,7 @@ public SlimeWorld readVanillaWorld(File worldDir, String worldName, SlimeLoader
         SlimeWorld world;
 
         try {
-            world = this.reader.readFromData(AnvilImportData.legacy(worldDir, worldName, loader));
+            world = this.reader.readFromData(new AnvilImportData(worldDir.toPath(), worldName, loader, extraRegionFolders));
         } catch (RuntimeException e) {
             if (e.getCause() == null) {
                 throw e;

core/build.gradle.kts

@@ -8,6 +8,7 @@ dependencies {
     compileOnly(project(":api"))
     compileOnly(paperApi())
     implementation(libs.zstd)
+    compileOnly(libs.lz4)
 }
 
 publishConfiguration {

core/src/main/java/com/infernalsuite/asp/serialization/anvil/AnvilImportData.java

@@ -1,12 +1,18 @@
 package com.infernalsuite.asp.serialization.anvil;
 
 import com.infernalsuite.asp.api.loaders.SlimeLoader;
+import com.infernalsuite.asp.api.world.ExtraRegionFolder;
 import org.jetbrains.annotations.Nullable;
 
 import java.io.File;
 import java.nio.file.Path;
+import java.util.List;
 
-public record AnvilImportData(Path worldDir, String newName, @Nullable SlimeLoader loader) {
+public record AnvilImportData(Path worldDir, String newName, @Nullable SlimeLoader loader, List<ExtraRegionFolder> extraRegionFolders) {
+
+    public AnvilImportData(Path worldDir, String newName, @Nullable SlimeLoader loader) {
+        this(worldDir, newName, loader, List.of());
+    }
 
     public static AnvilImportData legacy(File worldDir, String newName, @Nullable SlimeLoader loader) {
         return new AnvilImportData(worldDir.toPath(), newName, loader);

core/src/main/java/com/infernalsuite/asp/serialization/anvil/AnvilWorldReader.java

@@ -4,6 +4,7 @@
 import com.infernalsuite.asp.api.SlimeDataConverter;
 import com.infernalsuite.asp.api.exceptions.InvalidWorldException;
 import com.infernalsuite.asp.api.utils.NibbleArray;
+import com.infernalsuite.asp.api.world.ExtraRegionFolder;
 import com.infernalsuite.asp.api.world.SlimeChunk;
 import com.infernalsuite.asp.api.world.SlimeChunkSection;
 import com.infernalsuite.asp.api.world.SlimeWorld;
@@ -169,6 +170,13 @@ public SlimeWorld readFromData(AnvilImportData importData) {
                 throw new InvalidWorldException(environmentDir);
             }
 
+            for (ExtraRegionFolder extraRegionFolder : importData.extraRegionFolders()) {
+                Path extraRegionDir = worldDir.resolve(extraRegionFolder.directory());
+                if (Files.isDirectory(extraRegionDir)) {
+                    ExtraRegionReader.attachTo(extraRegionDir, extraRegionFolder.extraDataKey(), chunks);
+                }
+            }
+
             propertyMap.setValue(SlimeProperties.SPAWN_X, spawnX);
             propertyMap.setValue(SlimeProperties.SPAWN_Y, spawnY);
             propertyMap.setValue(SlimeProperties.SPAWN_Z, spawnZ);

core/src/main/java/com/infernalsuite/asp/serialization/anvil/ExtraRegionReader.java

@@ -0,0 +1,154 @@
+package com.infernalsuite.asp.serialization.anvil;
+
+import com.github.luben.zstd.ZstdInputStream;
+import com.infernalsuite.asp.api.world.SlimeChunk;
+import com.infernalsuite.asp.util.Util;
+import it.unimi.dsi.fastutil.longs.Long2ObjectMap;
+import net.jpountz.lz4.LZ4BlockInputStream;
+import net.kyori.adventure.nbt.BinaryTagTypes;
+import net.kyori.adventure.nbt.ByteArrayBinaryTag;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.InflaterInputStream;
+
+final class ExtraRegionReader {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(ExtraRegionReader.class);
+
+    private static final Pattern REGION_FILE = Pattern.compile("^r\\.(-?\\d+)\\.(-?\\d+)\\.mca$");
+
+    private static final int SECTOR_SIZE = 4096;
+    private static final int HEADER_SIZE = 2 * SECTOR_SIZE;
+    private static final int CHUNK_HEADER_SIZE = 5;
+    private static final int EXTERNAL_STREAM_FLAG = 0b1000_0000;
+
+    private ExtraRegionReader() {
+    }
+
+    static void attachTo(Path directory, String extraDataKey, Long2ObjectMap<SlimeChunk> chunks) throws IOException {
+        int attached = 0;
+        int orphaned = 0;
+
+        try (var stream = Files.newDirectoryStream(directory, path -> path.toString().endsWith(".mca"))) {
+            for (final Path path : stream) {
+                Matcher matcher = REGION_FILE.matcher(path.getFileName().toString());
+                if (!matcher.matches()) continue;
+
+                byte[] bytes = Files.readAllBytes(path);
+                if (bytes.length < HEADER_SIZE) continue;
+
+                LOGGER.info("Loading {} region file {}...", extraDataKey, path.getFileName());
+
+                ByteBuffer region = ByteBuffer.wrap(bytes);
+                int regionX = Integer.parseInt(matcher.group(1));
+                int regionZ = Integer.parseInt(matcher.group(2));
+
+                for (int index = 0; index < 1024; index++) {
+                    int sectorInfo = region.getInt(index * 4);
+                    if (sectorInfo == 0) continue;
+
+                    int chunkX = (regionX << 5) + (index & 31);
+                    int chunkZ = (regionZ << 5) + (index >> 5);
+
+                    SlimeChunk chunk = chunks.get(Util.chunkPosition(chunkX, chunkZ));
+                    if (chunk == null) {
+                        orphaned++;
+                        continue;
+                    }
+
+                    byte[] data = readChunk(directory, region, sectorInfo, chunkX, chunkZ);
+                    if (data == null) continue;
+
+                    chunk.getExtraData().put(extraDataKey, ByteArrayBinaryTag.byteArrayBinaryTag(data));
+                    attached++;
+                }
+            }
+        }
+
+        LOGGER.info("Attached {} data to {} chunks", extraDataKey, attached);
+        if (orphaned > 0) {
+            LOGGER.warn("Dropped {} data of {} chunks that are not part of the imported world", extraDataKey, orphaned);
+        }
+    }
+
+    private static byte[] readChunk(Path directory, ByteBuffer region, int sectorInfo, int chunkX, int chunkZ) throws IOException {
+        int offset = ((sectorInfo >> 8) & 0xFFFFFF) * SECTOR_SIZE;
+        if (offset < HEADER_SIZE || offset + CHUNK_HEADER_SIZE > region.capacity()) {
+            LOGGER.warn("Chunk at {},{} points outside of the region file", chunkX, chunkZ);
+            return null;
+        }
+
+        int length = region.getInt(offset) - 1;
+        int flags = region.get(offset + 4) & 0xFF;
+
+        InputStream source;
+        if ((flags & EXTERNAL_STREAM_FLAG) != 0) {
+            Path external = directory.resolve("c." + chunkX + "." + chunkZ + ".mcc");
+            if (!Files.isRegularFile(external)) {
+                LOGGER.warn("Chunk at {},{} refers to a missing external file {}", chunkX, chunkZ, external.getFileName());
+                return null;
+            }
+            source = Files.newInputStream(external);
+        } else if (length < 0 || offset + CHUNK_HEADER_SIZE + length > region.capacity()) {
+            LOGGER.warn("Chunk at {},{} has a truncated stream", chunkX, chunkZ);
+            return null;
+        } else {
+            source = new ByteArrayInputStream(region.array(), offset + CHUNK_HEADER_SIZE, length);
+        }
+
+        InputStream decompressor = decompress(flags & 0b111, source);
+        if (decompressor == null) {
+            source.close();
+            LOGGER.warn("Chunk at {},{} uses an unsupported compression method {}", chunkX, chunkZ, flags & 0b111);
+            return null;
+        }
+
+        byte[] data;
+        try (decompressor) {
+            data = decompressor.readAllBytes();
+        }
+
+        return switch ((flags >> 3) & 0b1111) {
+            case 0 -> stripRootName(data, chunkX, chunkZ);
+            case 1 -> data;
+            default -> {
+                LOGGER.warn("Chunk at {},{} uses an unsupported format version {}", chunkX, chunkZ, (flags >> 3) & 0b1111);
+                yield null;
+            }
+        };
+    }
+
+    private static InputStream decompress(int compression, InputStream source) throws IOException {
+        return switch (compression) {
+            case 1 -> source;
+            case 2 -> new InflaterInputStream(source);
+            case 3 -> new GZIPInputStream(source);
+            case 4 -> LZ4BlockInputStream.newBuilder().build(source);
+            case 5 -> new ZstdInputStream(source);
+            default -> null;
+        };
+    }
+
+    private static byte[] stripRootName(byte[] data, int chunkX, int chunkZ) {
+        if (data.length < 3 || data[0] != BinaryTagTypes.COMPOUND.id()) {
+            LOGGER.warn("Chunk at {},{} does not hold a compound tag", chunkX, chunkZ);
+            return null;
+        }
+
+        int payload = 3 + (((data[1] & 0xFF) << 8) | (data[2] & 0xFF));
+        byte[] nameless = new byte[data.length - payload + 1];
+        nameless[0] = data[0];
+        System.arraycopy(data, payload, nameless, 1, nameless.length - 1);
+        return nameless;
+    }
+}

gradle/libs.versions.toml

@@ -14,6 +14,7 @@ kotlin = "1.9.25"
 lettuce = "6.5.1.RELEASE"
 lombok = "1.18.36"
 lombok-plugin = "8.11"
+lz4 = "1.10.1"
 mongo = "5.2.1"
 paperweight = "2.0.0-SNAPSHOT"
 plugin-yml-paper = "0.6.0"
@@ -48,6 +49,8 @@ configurate-yaml = { module = "org.spongepowered:configurate-yaml", version.ref
 hikari = { module = "com.zaxxer:HikariCP", version.ref = "hikari" }
 lettuce = { module = "io.lettuce:lettuce-core", version.ref = "lettuce" }
 lombok = { module = "org.projectlombok:lombok", version.ref = "lombok" }
+lz4 = { module = "at.yawk.lz4:lz4-java", version.ref = "lz4" }
 mongo = { module = "org.mongodb:mongodb-driver-sync", version.ref = "mongo" }
 slf4j-api = { module = "org.slf4j:slf4j-api", version.ref = "slf4j" }
+slf4j-simple = { module = "org.slf4j:slf4j-simple", version.ref = "slf4j" }
 zstd = { module = "com.github.luben:zstd-jni", version.ref = "zstd" }

importer/build.gradle.kts

@@ -7,6 +7,8 @@ dependencies {
     implementation(project(":api"))
     implementation(project(":core"))
     implementation(project(":aspaper-api"))
+    implementation(libs.lz4)
+    runtimeOnly(libs.slf4j.simple)
 }
 
 tasks {
@@ -17,7 +19,10 @@ tasks {
     }
     shadowJar {
         archiveClassifier.set("")
-        minimize()
+        minimize {
+            exclude(dependency("at.yawk.lz4:.*"))
+            exclude(dependency("org.slf4j:.*"))
+        }
     }
     assemble {
         dependsOn(shadowJar)

importer/src/main/java/com/infernalsuite/asp/importer/SWMImporter.java

@@ -1,12 +1,14 @@
 package com.infernalsuite.asp.importer;
 
+import com.infernalsuite.asp.api.world.ExtraRegionFolder;
 import com.infernalsuite.asp.serialization.anvil.AnvilImportData;
 import com.infernalsuite.asp.serialization.anvil.AnvilWorldReader;
 import com.infernalsuite.asp.serialization.slime.SlimeSerializer;
 
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Scanner;
@@ -15,7 +17,7 @@ public class SWMImporter {
 
     public static void main(String[] args) {
         if (args.length == 0) {
-            System.err.println("Usage: java -jar slimeworldmanager-importer.jar <path-to-world-folder> [--accept] [--print-error]");
+            System.err.println("Usage: java -jar slimeworldmanager-importer.jar <path-to-world-folder> [--accept] [--print-error] [--extra-region <folder>[=<chunk-data-key>]]");
             return;
         }
 
@@ -26,6 +28,18 @@ public static void main(String[] args) {
         boolean hasAccepted = argList.contains("--accept");
         boolean printErrors = argList.contains("--print-error");
 
+        List<ExtraRegionFolder> extraRegionFolders = new ArrayList<>();
+        for (int i = 1; i < args.length; i++) {
+            if (!args[i].equals("--extra-region")) continue;
+
+            if (++i == args.length) {
+                System.err.println("--extra-region requires a <folder>[=<chunk-data-key>] value.");
+                return;
+            }
+
+            extraRegionFolders.add(ExtraRegionFolder.parse(args[i]));
+        }
+
         if(!hasAccepted) {
             System.out.println("**** WARNING ****");
             System.out.println("The Slime Format is meant to be used on tiny maps, not big survival worlds. It is recommended " +
@@ -43,13 +57,18 @@ public static void main(String[] args) {
             }
         }
 
-        importWorld(worldDir, outputFile, printErrors);
+        importWorld(worldDir, outputFile, printErrors, extraRegionFolders);
     }
 
     public static void importWorld(File worldDir, File outputFile, boolean shouldPrintDebug) {
+        importWorld(worldDir, outputFile, shouldPrintDebug, List.of());
+    }
+
+    public static void importWorld(File worldDir, File outputFile, boolean shouldPrintDebug, List<ExtraRegionFolder> extraRegionFolders) {
         try {
             outputFile.createNewFile();
-            Files.write(outputFile.toPath(), SlimeSerializer.serialize(AnvilWorldReader.INSTANCE_NO_CONVERSION.readFromData(AnvilImportData.legacy(worldDir, outputFile.getName(), null))));
+            AnvilImportData importData = new AnvilImportData(worldDir.toPath(), outputFile.getName(), null, extraRegionFolders);
+            Files.write(outputFile.toPath(), SlimeSerializer.serialize(AnvilWorldReader.INSTANCE_NO_CONVERSION.readFromData(importData)));
         } catch (IndexOutOfBoundsException ex) {
             System.err.println("Oops, it looks like the world provided is too big to be imported. " +
                     "Please trim it by using the MCEdit tool and try again.");