diff --git a/.github/workflows/coop-deployment-scripts.yml b/.github/workflows/coop-deployment-scripts.yml new file mode 100644 index 00000000..a72d6ccc --- /dev/null +++ b/.github/workflows/coop-deployment-scripts.yml @@ -0,0 +1,46 @@ +name: Coop deployment scripts + +on: + push: + paths: + - 'apps/faf-legacy-deployment/scripts/**' + - '.github/workflows/coop-deployment-scripts.yml' + pull_request: + paths: + - 'apps/faf-legacy-deployment/scripts/**' + - '.github/workflows/coop-deployment-scripts.yml' + +# the job builds and runs repository controlled Kotlin, so it gets nothing but read access +permissions: + contents: read + +jobs: + verify: + + runs-on: ubuntu-latest + container: + # same image the deployment CronJobs use + image: gradle:9.4-jdk21 + + steps: + - uses: actions/checkout@v6 + with: + # nothing here pushes, so the job token has no business staying in .git/config + persist-credentials: false + + - name: Check out the coop missions + uses: actions/checkout@v6 + with: + repository: FAForever/faf-coop-maps + path: faf-coop-maps + persist-credentials: false + + - name: Compile + working-directory: apps/faf-legacy-deployment/scripts + run: gradle --no-daemon compileKotlin + + - name: Round trip the path fixer over every mission map + working-directory: apps/faf-legacy-deployment/scripts + env: + MAPS_REPO: ${{ github.workspace }}/faf-coop-maps + run: gradle --no-daemon verifyScmapFixer diff --git a/apps/faf-legacy-deployment/scripts/CoopMapDeployer.kt b/apps/faf-legacy-deployment/scripts/CoopMapDeployer.kt index 748c7b59..5e9d1808 100644 --- a/apps/faf-legacy-deployment/scripts/CoopMapDeployer.kt +++ b/apps/faf-legacy-deployment/scripts/CoopMapDeployer.kt @@ -7,13 +7,16 @@ import com.faforever.FafDatabase import com.faforever.GitRepo import com.faforever.Log import com.faforever.extractChecksumsFromZip +import com.faforever.fixScmapPaths import com.faforever.generateChecksums +import com.faforever.needsPathFix import org.apache.commons.compress.archivers.zip.ZipArchiveEntry import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream import org.slf4j.LoggerFactory import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths +import java.nio.file.StandardCopyOption import kotlin.io.path.copyTo import kotlin.io.path.createDirectories import kotlin.io.path.isDirectory @@ -21,6 +24,8 @@ import kotlin.io.path.isRegularFile import kotlin.io.path.readBytes import kotlin.io.path.readText import kotlin.io.path.walk +import kotlin.system.exitProcess +import kotlin.toString private val log = LoggerFactory.getLogger("coop-maps-updater") @@ -151,9 +156,17 @@ private fun processCoopMap( val newVersion = currentVersion + 1 log.info("$map updated → v$newVersion") + verifyRelease(map, newVersion, files, tmp) + if (!simulate) { val finalZip = Path.of(mapsDir, map.zipName(newVersion)) - createZip(map, newVersion, files, tmp, finalZip) + val partialZip = Path.of(mapsDir, "${map.zipName(newVersion)}.part") + try { + createZip(map, newVersion, files, tmp, partialZip) + Files.move(partialZip, finalZip, StandardCopyOption.ATOMIC_MOVE) + } finally { + Files.deleteIfExists(partialZip) + } db.update(map, newVersion) } } finally { @@ -178,24 +191,102 @@ private fun generateChecksumsForMap( /** * Get file content with path rewriting for text files. + * + * Text files are read and written as ISO-8859-1. Latin-1 maps every byte to exactly one + * char and back, and the replacements below only ever touch ASCII, so a file comes out + * byte identical no matter what it is really encoded in. Reading as UTF-8 would turn + * anything that is not valid UTF-8 into U+FFFD and write that into the release instead. + * Every text file in the missions is valid UTF-8 today, so this changes no checksum - it + * only keeps the first file with a Latin-1 accent in it from being corrupted silently. */ -private fun getFileContent(file: Path, map: CoopMap, version: Int): ByteArray { - return if (file.isTextFile()) { - var text = file.readText() - .replace( - "/maps/${map.folderName}/", - "/maps/${map.folderName(version)}/", - ignoreCase = true, - ) - if (file.toString().endsWith("_scenario.lua")) { - text = text.replace(Regex("""(map_version\s*=\s*)\d+"""), "$1$version") +private fun getFileContent(file: Path, map: CoopMap, version: Int): ByteArray = + when { + file.isScmapFile() -> { + val bytes = file.readBytes() + // Only missions whose map references assets in their own folder need the version + // inserted. Everything else - including the placeholder .scmap files of the missions + // that use a base game map - is passed through and never parsed. + if (bytes.needsPathFix(map.folderName)) { + fixScmapPaths(bytes, map.folderName, version).bytes + } else { + bytes + } + } + file.isTextFile() -> { + var text = file.readText(Charsets.ISO_8859_1) + .replace( + "/maps/${map.folderName}/", + "/maps/${map.folderName(version)}/", + ignoreCase = true, + ) + if (file.toString().endsWith("_scenario.lua", ignoreCase = true)) { + text = text.replace(Regex("""(map_version\s*=\s*)\d+"""), "$1$version") + } + text.toByteArray(Charsets.ISO_8859_1) } - text.toByteArray() - } else { - file.readBytes() + else -> file.readBytes() + } + +/** + * Checks that every path the release points at actually exists in the release. + * + * The paths rewritten inside a .scmap are checked hard: if one of them does not resolve to + * a file that ends up in the zip, the mission is not deployed at all. That is the case the + * whole path rewriting exists for, and getting it wrong ships a map with missing textures + * that nothing else would notice. + * + * References in text files are only reported. Some of them have been broken for years - + * typos in comment headers - and failing on those would block releases for cosmetic reasons. + * + * @throws IllegalStateException if a rewritten map path does not resolve + */ +private fun verifyRelease(map: CoopMap, version: Int, files: List, base: Path) { + // Everything below is compared case insensitively, and it has to be: 28 of the paths + // embedded in the map files point at lower case names while the files themselves are + // mixed case. That is in the missions as committed, it predates this deployment, and + // those maps load - a case sensitive comparison would refuse releases that demonstrably + // work. What the rewriting must not do is add mismatches on top of that; see addVersion. + val prefix = "/maps/${map.folderName(version)}/".lowercase() + val shipped = files + .map { prefix + base.relativize(it).toString().replace("\\", "/").lowercase() } + .toSet() + + // over capture is possible when a path is read out of binary data, so a reference counts + // as resolved when it starts with a shipped file + fun resolves(reference: String) = + shipped.any { reference.lowercase() == it || reference.lowercase().startsWith(it) } + + val broken = mutableListOf() + + files.forEach { file -> + val relative = base.relativize(file).toString().replace("\\", "/") + + if (file.isScmapFile()) { + val bytes = file.readBytes() + if (bytes.needsPathFix(map.folderName)) { + fixScmapPaths(bytes, map.folderName, version).rewritten + .filterNot(::resolves) + .forEach { broken += "$relative points at $it, which is not in the release" } + } + } else if (file.isTextFile()) { + val text = String(getFileContent(file, map, version), Charsets.ISO_8859_1) + MAP_REFERENCE.findAll(text) + .map { it.value } + .filter { it.lowercase().startsWith("/maps/${map.folderName.lowercase()}") } + .filterNot(::resolves) + .distinct() + .forEach { log.warn("$map: $relative points at $it, which is not in the release") } + } + } + + check(broken.isEmpty()) { + broken.forEach { log.error("$map: $it") } + "$map: ${broken.size} rewritten map path(s) do not resolve, not deploying this mission" } } +private val MAP_REFERENCE = Regex("""/maps/[^"'\s,)]+""", RegexOption.IGNORE_CASE) + private fun createZip( map: CoopMap, version: Int, @@ -236,7 +327,9 @@ private fun createZip( } } -private fun Path.isTextFile() = listOf(".md", ".lua", ".json", ".txt").any { toString().endsWith(it) } +private fun Path.isTextFile() = listOf(".md", ".lua", ".json", ".txt").any { toString().endsWith(it, ignoreCase = true) } + +private fun Path.isScmapFile() = toString().endsWith(".scmap", ignoreCase = true) fun main(args: Array) { Log.init() @@ -258,13 +351,23 @@ fun main(args: Array) { gitRef = GIT_REF, ).checkout() + val failed = mutableListOf() + CoopMapDatabase(dryRun = DRYRUN).use { db -> coopMaps.forEach { try { processCoopMap(db, it, DRYRUN, WORKDIR, MAP_DIR) } catch (e: Exception) { - log.warn("Failed processing $it", e) + failed += it + log.error("Failed processing $it", e) } } } + + // one mission failing must not stop the others, but it may not pass for a successful run + // either - a refused release is only visible in the logs otherwise + if (failed.isNotEmpty()) { + log.error("{} mission(s) were not deployed: {}", failed.size, failed.joinToString { it.folderName }) + exitProcess(1) + } } diff --git a/apps/faf-legacy-deployment/scripts/ScmapPathFixer.kt b/apps/faf-legacy-deployment/scripts/ScmapPathFixer.kt new file mode 100644 index 00000000..e31f0d4e --- /dev/null +++ b/apps/faf-legacy-deployment/scripts/ScmapPathFixer.kt @@ -0,0 +1,387 @@ +@file:Suppress("PackageDirectoryMismatch") + +package com.faforever + +import org.slf4j.LoggerFactory +import java.io.ByteArrayOutputStream + +private val log = LoggerFactory.getLogger("scmap-path-fixer") + +/** Every .scmap file starts with these 16 bytes. */ +private val SCMAP_HEADER = byteArrayOf( + 0x4D, 0x61, 0x70, 0x1A, // "Map\x1A" + 0x02, 0x00, 0x00, 0x00, // 2 + 0xED.toByte(), 0xFE.toByte(), 0xEF.toByte(), 0xBE.toByte(), // 0xBEEFFEED, little endian + 0x02, 0x00, 0x00, 0x00, // 2 +) + +fun ByteArray.isScmap(): Boolean = + size >= SCMAP_HEADER.size && SCMAP_HEADER.indices.all { this[it] == SCMAP_HEADER[it] } + +/** Case insensitive raw byte search for an ASCII needle. */ +fun ByteArray.containsAscii(needle: String): Boolean { + val bytes = needle.lowercase().toByteArray(Charsets.ISO_8859_1) + if (bytes.size > size) return false + outer@ for (i in 0..size - bytes.size) { + for (j in bytes.indices) { + var b = this[i + j].toInt() and 0xFF + if (b in 0x41..0x5A) b += 0x20 // ASCII toLowerCase + if (b != (bytes[j].toInt() and 0xFF)) continue@outer + } + return true + } + return false +} + +/** + * Whether the map references its own folder, so we only parse map files that actually need + * the fix. Missions whose .scmap contains no such path (and the placeholder files that are + * not maps at all) are passed through untouched. + * + * Deliberately without a trailing slash, unlike the leftover scan in [fixScmapPaths]. This + * one has to match `/maps/.v0003/...` as well - a self reference that already + * carries a version - so that such a file reaches the parser, where the old suffix is + * stripped instead of a second one being stacked on top. The leftover scan wants the exact + * opposite: it asserts that no *unversioned* reference survived, so it needs the trailing + * slash to not match the versioned form it just wrote. + */ +fun ByteArray.referencesOwnMapFolder(folderName: String) = containsAscii("/maps/$folderName") + +/** A .scmap only needs rewriting when it is one and it points into its own folder. */ +fun ByteArray.needsPathFix(folderName: String) = isScmap() && referencesOwnMapFolder(folderName) + +/** + * Result of [fixScmapPaths]. + * + * @property bytes the rewritten map file + * @property rewritten every path that received the version, as written into [bytes] + * @property skipped paths inside /maps that lead to another folder and were left alone + */ +data class ScmapPathFixResult( + val bytes: ByteArray, + val rewritten: List, + val skipped: List, +) + +/** + * Rewrites the map folder segment of every path embedded in a .scmap so that it carries + * the release version: + * + * /maps/faf_coop_operation_blockade/env/layers/sand.dds + * -> /maps/faf_coop_operation_blockade.v0004/env/layers/sand.dds + * + * Only paths that point at [folderName] itself are touched. Paths pointing somewhere else + * are left alone and reported in [ScmapPathFixResult.skipped] - several missions deliberately + * reference a base game map + * (`/maps/X1CA_001/X1CA_001.scmap`) or another map's texture, and versioning those would + * break them. + * + * A .scmap is tightly packed with no field names, so every field has to be read and + * written back in the exact same order. Most strings are null terminated; decal texture + * paths are the exception and carry a leading int with their length. Arrays of structs + * (decals, waves, props) start with an int count. Which sections exist depends on the map + * file version - the coop missions use 53, 56 and 60. + * + * Ported from speed2's `sc_map_parser.gd` (`fix_paths`). + * + * @param version release version, or a negative value to copy the file without changes + * (used as a self check: the parser then has to reproduce the input byte + * for byte) + * @throws IllegalArgumentException if the file is not a .scmap + * @throws IllegalStateException if the result fails verification + */ + +fun fixScmapPaths(bytes: ByteArray, folderName: String, version: Int): ScmapPathFixResult { + require(bytes.isScmap()) { "$folderName: not a .scmap file, header mismatch" } + + val suffix = if (version >= 0) ".v%04d".format(version) else "" + val rewriter = ScmapRewriter(bytes, folderName, suffix) + val rewrittenBytes = rewriter.rewrite() + val result = ScmapPathFixResult(rewrittenBytes, rewriter.rewritten.toList(), rewriter.skipped.distinct()) + + if (suffix.isEmpty()) return result + + result.skipped.forEach { + log.warn("$folderName: path leads outside the mission folder, left unchanged: $it") + } + log.info( + "$folderName: rewrote {} path(s) in the map file, {} byte(s) added", + result.rewritten.size, rewrittenBytes.size - bytes.size + ) + + // the file may only have grown by exactly what went into the paths - anything else means + // the structural pass lost or invented bytes somewhere between them + check(rewrittenBytes.size - bytes.size == rewriter.addedBytes) { + "$folderName: byte delta ${rewrittenBytes.size - bytes.size} does not match the " + + "${rewriter.addedBytes} byte(s) added to paths" + } + // no unversioned reference to the mission folder may survive. This is the one check that + // catches a path the parser never recognised as a path in the first place - the delta + // accounting and the round trip below are both blind to a rewrite that simply did not + // happen, which is exactly the bug this whole class exists to fix. + check(!rewrittenBytes.containsAscii("/maps/$folderName/")) { + "$folderName: the rewritten .scmap still contains unversioned /maps/$folderName/ " + + "path(s), refusing to ship it" + } + // the rewritten file has to parse again and come out byte identical + val verified = ScmapRewriter(rewrittenBytes, folderName, "").rewrite() + check(verified.contentEquals(rewrittenBytes)) { + "$folderName: rewritten .scmap does not round trip, refusing to ship it" + } + return result +} + +private class ScmapRewriter( + private val src: ByteArray, + folderName: String, + private val suffix: String, +) { + private val folder = folderName.lowercase() + private val out = ByteArrayOutputStream(src.size + 1024) + private var pos = 0 + + val rewritten = mutableListOf() + val skipped = mutableListOf() + var addedBytes = 0 + private set + + fun rewrite(): ByteArray { + copy(16) // header + copy(14) // float size x, y + 6 padding bytes + transferSizedChunk() // preview image + + val version = readInt() + val width = readInt() + val height = readInt() + writeInt(version) + writeInt(width) + writeInt(height) + copy(4) // height scale + + copy((width + 1) * (height + 1) * 2) // heightmap + if (version >= 56) copy(1) // padding after the heightmap + + transferString() // shader name, not a path + transferString(true) // editor background + transferString(true) // skycube + + if (version >= 56) { + val cubemaps = readInt() + writeInt(cubemaps) + repeat(cubemaps) { + transferString() // name + transferString(true) // path + } + } else { + transferString(true) // old format has a single cubemap + } + + copy(23 * 4) // lighting + copy(1 + 23 * 4) // water flag + water settings + transferString(true) // water cubemap + transferString(true) // water ramp + copy(4 * 4) // wave normal repeats + repeat(4) { // wave textures + copy(8) // movement + transferString(true) + } + + val waves = readInt() + writeInt(waves) + repeat(waves) { + transferString() // texture name, not a path + transferString() // ramp name, not a path + copy(17 * 4) + } + + if (version < 56) { + transferString() // always "No Tileset" + val tilesets = readInt() // always 6 + writeInt(tilesets) + repeat(tilesets) { + transferString(true) // albedo + transferString(true) // normal + copy(2 * 4) // scales + } + } else { + copy(6 * 4) // minimap + if (version > 56) copy(4) // unknown + repeat(19) { // 10 albedo + 9 normal + transferString(true) + copy(4) // scale + } + } + + copy(8) // 2 unknown values + + val decals = readInt() + writeInt(decals) + repeat(decals) { + copy(8) // id + type + val textures = readInt() + writeInt(textures) + repeat(textures) { transferSizedString() } + copy(12 * 4) // 11 floats + 1 int + } + + val decalGroups = readInt() + writeInt(decalGroups) + repeat(decalGroups) { + copy(4) // id + transferString() // name, not a path + val ids = readInt() + writeInt(ids) + if (ids > 0) copy(ids * 4) + } + + copy(8) // int width + height + + val normalMaps = readInt() + writeInt(normalMaps) + repeat(normalMaps) { transferSizedChunk() } + + if (version < 56) copy(4) // unknown in the old format + + transferSizedChunk() // texture mask low + if (version >= 56) transferSizedChunk() // texture mask high + + val waterMaps = readInt() + writeInt(waterMaps) + repeat(waterMaps) { transferSizedChunk() } + + val halfSize = (width / 2) * (height / 2) + repeat(3) { copy(halfSize) } // water foam, flatness, depth bias + copy(width * height) // terrain types + + if (version <= 52) copy(2) // unknown in the oldest format + + if (version >= 60) { + copy(16 * 4) // skybox settings + transferString(true) // albedo + transferString(true) // glow + val planets = readInt() + writeInt(planets) + repeat(planets) { copy(10 * 4) } + copy(3 + 4 * 4) // sky mid color + cirrus + transferString(true) // cirrus texture + val cirrusLayers = readInt() + writeInt(cirrusLayers) + repeat(cirrusLayers) { copy(5 * 4) } + copy(4) // one more setting + } + + val props = readInt() + writeInt(props) + repeat(props) { + transferString(true) // blueprint path + copy(15 * 4) + } + + if (pos < src.size) { + log.warn("$folder: {} trailing byte(s) after the props, copied unchanged", src.size - pos) + copy(src.size - pos) + } + return out.toByteArray() + } + + /** + * If [path] points into this mission's own folder inside /maps, the version is added to + * the folder name. Only the `/maps/.vNNNN` head is lower cased; everything + * after it keeps the casing it had. + * + * `fix_paths` lower cases the whole path, and this deliberately does not. Two of the + * four affected missions reference their decals in exactly the casing the files carry + * on disk - `env/decals/FAF_Coop_Operation_Golden_Crystals_nm.dds` and the three like + * it - and lower casing those introduces a mismatch that is not in the source data. + * The other 28 paths already point at lower case names while the files are mixed case, + * so a lower cased tail buys nothing there either. + * + * Everything else is returned byte for byte as it came in, including paths under /maps + * that lead to another folder. Those are shipped unparsed today and work, so there is + * nothing to gain by touching them - and something to lose: this method is also handed + * whatever a mis-framed [transferString] believes to be a string, and lower casing that + * would silently flip bytes inside binary data without changing the length, which no + * check in [fixScmapPaths] could catch. + */ + private fun addVersion(path: String): String { + if (suffix.isEmpty()) return path + val match = MAP_PATH.matchEntire(path) ?: return path + val (prefix, segment, rest) = match.destructured + + val bare = segment.lowercase().replace(VERSIONED, "") + if (bare != folder) { + skipped += path + return path + } + + val fixed = (prefix + bare + suffix).lowercase() + rest + addedBytes += fixed.length - path.length + rewritten += fixed + return fixed + } + + /** Null terminated string. [isPath] marks the ones that may need the version. */ + private fun transferString(isPath: Boolean = false) { + val start = pos + while (pos < src.size && src[pos] != 0.toByte()) pos++ + val raw = String(src, start, pos - start, Charsets.ISO_8859_1) + pos++ // null terminator + val value = if (isPath) addVersion(raw) else raw + out.write(value.toByteArray(Charsets.ISO_8859_1)) + out.write(0) + } + + /** Decal texture paths are not null terminated but prefixed with their length. */ + private fun transferSizedString() { + val length = readInt() + val raw = String(src, pos, length, Charsets.ISO_8859_1) + pos += length + val encoded = addVersion(raw).toByteArray(Charsets.ISO_8859_1) + writeInt(encoded.size) + out.write(encoded) + } + + /** Chunk of bytes whose length is read from the file, used for embedded textures. */ + private fun transferSizedChunk() { + val length = readInt() + writeInt(length) + if (length > 0) copy(length) + } + + private fun readInt(): Int { + val v = (src[pos].toInt() and 0xFF) or + ((src[pos + 1].toInt() and 0xFF) shl 8) or + ((src[pos + 2].toInt() and 0xFF) shl 16) or + ((src[pos + 3].toInt() and 0xFF) shl 24) + pos += 4 + return v + } + + private fun writeInt(value: Int) { + out.write(value and 0xFF) + out.write((value ushr 8) and 0xFF) + out.write((value ushr 16) and 0xFF) + out.write((value ushr 24) and 0xFF) + } + + private fun copy(length: Int) { + out.write(src, pos, length) + pos += length + } + + private companion object { + val VERSIONED = Regex("""\.v\d{4}$""") + + /** + * `/maps//`, with the three parts captured separately. + * + * The leading slash is required, so this matches exactly what + * [referencesOwnMapFolder] searches for. All 40 paths embedded in the missions carry + * it; the slashless form was only ever accepted because `fix_paths` splits on `/` + * discarding empty segments, which makes `maps/x/y` and `/maps/x/y` indistinguishable + * to it. Accepting a form the gate does not detect means the gate can wave a file + * through that the rewriter would have changed. + */ + val MAP_PATH = Regex("""^(/maps/)([^/]+)(/.*)$""", RegexOption.IGNORE_CASE) + } +} diff --git a/apps/faf-legacy-deployment/scripts/ScmapPathFixerCheck.kt b/apps/faf-legacy-deployment/scripts/ScmapPathFixerCheck.kt new file mode 100644 index 00000000..67f97dc7 --- /dev/null +++ b/apps/faf-legacy-deployment/scripts/ScmapPathFixerCheck.kt @@ -0,0 +1,149 @@ +@file:Suppress("PackageDirectoryMismatch") + +package com.faforever + +import java.io.File +import kotlin.system.exitProcess + +/** + * Runs [fixScmapPaths] over every map file in a checkout of the coop missions and fails if + * anything is off. Meant for CI, so a change to the path rewriting cannot quietly break a + * map file. + * + * Rewriting with a negative version has to reproduce the input byte for byte. That covers + * every mission, not only the handful that need the fix, but it is a weaker statement than + * it looks: with no suffix the rewriter is a byte copier by construction, so the identity + * check really only says that no read ran off the end of the file. The checks that carry + * weight are the ones [fixScmapPaths] makes on the rewritten output, and the detection + * cross check below. + * + * Point MAPS_REPO at a checkout of https://github.com/FAForever/faf-coop-maps. + */ +fun main() { + val repo = File(System.getenv("MAPS_REPO") ?: "/tmp/faf-coop-maps") + require(repo.isDirectory) { "MAPS_REPO does not point at a directory: $repo" } + + val maps = repo.walkTopDown() + .filter { it.isFile && it.name.lowercase().endsWith(".scmap") } + .sortedBy { it.path } + .toList() + require(maps.isNotEmpty()) { "no .scmap files found below $repo" } + + val failures = mutableListOf() + val rewrittenFolders = mutableListOf() + val versions = mutableMapOf() + var placeholders = 0 + var identical = 0 + + println("%-44s %-8s %-10s %s".format("mission", "format", "size", "result")) + + maps.forEach { file -> + val folder = file.parentFile.name + val bytes = file.readBytes() + + if (!bytes.isScmap()) { + placeholders++ + println("%-44s %-8s %-10s %s".format(folder, "-", bytes.size, "not a map file, skipped")) + return@forEach + } + + val notes = mutableListOf() + val version = formatVersion(bytes) + versions[version] = (versions[version] ?: 0) + 1 + + // The rewriter only knows the sections of the versions that are actually in this + // repo. Nothing new will appear here - maps are written as v60, occasionally still + // as v56 - so an unexpected version means a section boundary the rewriter has never + // been run against, and it has to be looked at rather than parsed on a guess. + if (version !in KNOWN_VERSIONS) { + failures += "$folder: unexpected .scmap version $version, the rewriter only " + + "covers ${KNOWN_VERSIONS.joinToString()}" + } + + try { + val copy = fixScmapPaths(bytes, folder, -1).bytes + if (copy.contentEquals(bytes)) { + identical++ + notes += "identical" + } else { + failures += "$folder: copy is ${copy.size} bytes, input is ${bytes.size}" + notes += "NOT IDENTICAL" + } + } catch (e: Exception) { + failures += "$folder: ${e.message}" + notes += "FAILED: ${e.message}" + } + + // referencesOwnMapFolder is both the gate and the thing under test here: if it ever + // regressed to false every check below would be skipped and CI would still pass. So + // it is compared against a second, independent implementation. + val detected = bytes.referencesOwnMapFolder(folder) + val expected = String(bytes, Charsets.ISO_8859_1).contains("/maps/$folder", ignoreCase = true) + if (detected != expected) { + failures += "$folder: referencesOwnMapFolder says $detected, a plain text search says $expected" + } + + if (detected) { + try { + // delta accounting, leftover scan and round trip are asserted in fixScmapPaths + val fix = fixScmapPaths(bytes, folder, 4) + rewrittenFolders += folder + notes += "${fix.rewritten.size} path(s) rewritten" + + fix.rewritten.filterNot { it.contains(".v0004/") } + .forEach { failures += "$folder: not versioned: $it" } + // only the /maps/.vNNNN head is lower cased, the rest of the path + // keeps the casing the map file had - see addVersion + fix.rewritten.map { it.substringBefore(".v0004/") } + .filter { it != it.lowercase() } + .forEach { failures += "$folder: map folder not lower cased: $it" } + fix.rewritten.filter { DOUBLE_VERSION.containsMatchIn(it) } + .forEach { failures += "$folder: version added twice: $it" } + } catch (e: Exception) { + failures += "$folder: ${e.message}" + notes += "FAILED: ${e.message}" + } + } + + println("%-44s %-8s %-10s %s".format(folder, "v$version", bytes.size, notes.joinToString(", "))) + } + + // a run in which nothing at all was rewritten proves nothing about the rewriting + if (rewrittenFolders.isEmpty()) { + failures += "no map file references its own folder, so nothing was rewritten - " + + "either the missions changed or the detection is broken" + } + + println() + println("$identical of ${maps.size - placeholders} map file(s) reproduced byte for byte") + println("${rewrittenFolders.size} map file(s) reference their own folder and were rewritten:") + rewrittenFolders.forEach { println(" $it") } + println("$placeholders placeholder file(s) skipped") + println("format versions: " + versions.toSortedMap().entries.joinToString { "v${it.key} x${it.value}" }) + + if (failures.isNotEmpty()) { + println() + failures.forEach { println("FAIL $it") } + println("${failures.size} failure(s)") + exitProcess(1) + } + println("all good") +} + +private val DOUBLE_VERSION = Regex("""\.v\d{4}\.v\d{4}""") + +/** The only .scmap format versions the coop missions contain. */ +private val KNOWN_VERSIONS = setOf(53, 56, 60) + +/** + * Map file format version, stored behind the preview image. A valid header is only 16 bytes, + * so a file can pass [isScmap] and still be too short to hold one - that is reported as an + * unknown version rather than taking the whole run down. + */ +private fun formatVersion(bytes: ByteArray): Int { + fun int(at: Int) = (bytes[at].toInt() and 0xFF) or + ((bytes[at + 1].toInt() and 0xFF) shl 8) or + ((bytes[at + 2].toInt() and 0xFF) shl 16) or + ((bytes[at + 3].toInt() and 0xFF) shl 24) + return runCatching { int(34 + int(30)) }.getOrDefault(-1) +} diff --git a/apps/faf-legacy-deployment/scripts/build.gradle.kts b/apps/faf-legacy-deployment/scripts/build.gradle.kts index 1dd3ffc9..7808b217 100644 --- a/apps/faf-legacy-deployment/scripts/build.gradle.kts +++ b/apps/faf-legacy-deployment/scripts/build.gradle.kts @@ -36,4 +36,12 @@ tasks.register("deployCoopMaps") { classpath = sourceSets.main.get().runtimeClasspath mainClass.set("com.faforever.coopmapdeployer.CoopMapDeployerKt") -} \ No newline at end of file +} + +tasks.register("verifyScmapFixer") { + group = "verification" + description = "Run the .scmap path fixer over every map in a faf-coop-maps checkout (MAPS_REPO)" + + classpath = sourceSets.main.get().runtimeClasspath + mainClass.set("com.faforever.ScmapPathFixerCheckKt") +}