diff --git a/Analyzer/AnalyzerTool.cs b/Analyzer/AnalyzerTool.cs index 918ef96..da8473c 100644 --- a/Analyzer/AnalyzerTool.cs +++ b/Analyzer/AnalyzerTool.cs @@ -5,8 +5,8 @@ using UnityDataTools.Analyzer.SQLite.Handlers; using UnityDataTools.Analyzer.SQLite.Parsers; using UnityDataTools.Analyzer.SQLite.Writers; -using UnityDataTools.Models; using UnityDataTools.FileSystem; +using UnityDataTools.Models; namespace UnityDataTools.Analyzer; @@ -131,6 +131,13 @@ public int Analyze(AnalyzeOptions options) Console.WriteLine(); Console.WriteLine($"Finalizing database. Successfully processed files: {countSuccess}, Failed files: {countFailures}, Files without TypeTrees: {countNoTypeTrees}, Ignored files: {countIgnored}"); + // Record data that can only be determined once every file has been processed (e.g. which + // referenced objects were never resolved) before the database is finalized. + foreach (var parser in parsers) + { + parser.FinalizeDatabase(); + } + writer.End(); foreach (var parser in parsers) { diff --git a/Analyzer/Resources/Init.sql b/Analyzer/Resources/Init.sql index 8e9480f..7cbde59 100644 --- a/Analyzer/Resources/Init.sql +++ b/Analyzer/Resources/Init.sql @@ -81,6 +81,25 @@ CREATE TABLE IF NOT EXISTS refs property_type INTEGER ); +-- One row per referenced object that analyze assigned an id to but never wrote an objects row +-- for (its serialized file was not part of the analyzed input). Written after all files are +-- processed, once we can tell which assigned ids never became objects. Common causes: analyzing a +-- partial set of AssetBundles, references into "unity default resources" (shipped without +-- TypeTrees), or a ContentDirectory build analyzed without ContentLayout.json. +-- Columns mirror the objects table so every object id resolves to exactly one of objects or +-- dangling_refs: +-- id - the analyzer object id (no row in objects has this id). +-- object_id - the target's local file id (LFID / m_PathID) within its serialized file. +-- serialized_file - references the serialized_files row of the (un-analyzed) file the target +-- lives in. That row has archive NULL and no objects of its own. +CREATE TABLE IF NOT EXISTS dangling_refs +( + id INTEGER, + object_id INTEGER, + serialized_file INTEGER, + PRIMARY KEY (id) +); + -- Resolves the property_path and property_type ids in the refs table to their string values. CREATE VIEW refs_view AS SELECT r.object, r.referenced_object, pn.name AS property_path, pt.name AS property_type @@ -88,6 +107,27 @@ FROM refs r INNER JOIN property_names pn ON r.property_path = pn.id INNER JOIN property_types pt ON r.property_type = pt.id; +-- Resolves dangling_refs to the source object(s) that reference each missing target, one row per +-- (referencing object -> dangling target) reference. Not populated when analyze is run with +-- --skip-references (neither refs nor dangling_refs are populated in that mode). +CREATE VIEW dangling_refs_view AS +SELECT + r.object AS source_id, + src_sf.name AS source_serialized_file, + src_o.object_id AS source_object_id, + pn.name AS property_path, + pt.name AS property_type, + d.id AS target_id, + tgt_sf.name AS target_serialized_file, + d.object_id AS target_object_id +FROM dangling_refs d +INNER JOIN refs r ON r.referenced_object = d.id +INNER JOIN objects src_o ON r.object = src_o.id +INNER JOIN serialized_files src_sf ON src_o.serialized_file = src_sf.id +INNER JOIN serialized_files tgt_sf ON d.serialized_file = tgt_sf.id +LEFT JOIN property_names pn ON r.property_path = pn.id +LEFT JOIN property_types pt ON r.property_type = pt.id; + CREATE VIEW object_view AS SELECT o.id, o.object_id, ab.name AS archive, sf.name AS serialized_file, t.name AS type, o.name, o.game_object, o.size, CASE @@ -157,8 +197,9 @@ INSERT INTO types (id, name) VALUES (-1, 'Scene'); -- 1 = normalized refs table (issue #44); 2 = renamed assets/asset_dependencies tables to -- assetbundle_assets/preload_dependencies (issue #82); 3 = renamed asset_bundles table to archives -- and the asset_bundle column/alias to archive (issue #68); 4 = build_report_packed_asset_contents_view --- type column changed from numeric id to type name (issue #55); databases produced before versioning report 0. -PRAGMA user_version = 4; +-- type column changed from numeric id to type name (issue #55); 5 = added dangling_refs table/view +-- (issue #85); databases produced before versioning report 0. +PRAGMA user_version = 5; PRAGMA synchronous = OFF; PRAGMA journal_mode = MEMORY; diff --git a/Analyzer/SQLite/Commands/SerializedFile/AddDanglingRef.cs b/Analyzer/SQLite/Commands/SerializedFile/AddDanglingRef.cs new file mode 100644 index 0000000..1af0907 --- /dev/null +++ b/Analyzer/SQLite/Commands/SerializedFile/AddDanglingRef.cs @@ -0,0 +1,29 @@ +using System.Collections.Generic; +using Microsoft.Data.Sqlite; +using UnityDataTools.Analyzer.SQLite.Commands; + +namespace UnityDataTools.Analyzer.SQLite.Commands.SerializedFile +{ + /* TABLE DEFINITION: + create table dangling_refs + ( + id INTEGER, + object_id INTEGER, + serialized_file INTEGER, + PRIMARY KEY (id) + ); + */ + internal class AddDanglingRef : AbstractCommand + { + protected override string TableName => "dangling_refs"; + + protected override string DDLSource => null; + + protected override Dictionary Fields => new() + { + { "id", SqliteType.Integer }, + { "object_id", SqliteType.Integer }, + { "serialized_file", SqliteType.Integer } + }; + } +} diff --git a/Analyzer/SQLite/Handlers/AssetBundleHandler.cs b/Analyzer/SQLite/Handlers/AssetBundleHandler.cs index 2d5552c..85e963d 100644 --- a/Analyzer/SQLite/Handlers/AssetBundleHandler.cs +++ b/Analyzer/SQLite/Handlers/AssetBundleHandler.cs @@ -68,6 +68,13 @@ public void Process(Context ctx, long objectId, RandomAccessReader reader, out s { if (!assetBundle.IsSceneAssetBundle) { + // Editor-only container entries (e.g. ShaderSubGraph, .preset) can appear in + // m_Container with a null PPtr (m_FileID 0 and m_PathID 0) because they have no + // runtime object. Skip them: resolving the null PPtr would allocate a phantom + // (file, 0) object id and record a bogus dangling reference to it. + if (asset.PPtr.FileId == 0 && asset.PPtr.PathId == 0) + continue; + var fileId = ctx.LocalToDbFileId[asset.PPtr.FileId]; var objId = ctx.ObjectIdProvider.GetId((fileId, asset.PPtr.PathId)); m_InsertCommand.Transaction = ctx.Transaction; diff --git a/Analyzer/SQLite/Handlers/ISQLiteHandler.cs b/Analyzer/SQLite/Handlers/ISQLiteHandler.cs index c5f8806..d266272 100644 --- a/Analyzer/SQLite/Handlers/ISQLiteHandler.cs +++ b/Analyzer/SQLite/Handlers/ISQLiteHandler.cs @@ -27,6 +27,12 @@ public interface ISQLiteFileParser : IDisposable void Init(SqliteConnection db); bool CanParse(string filename); void Parse(string filename); + + // Called once after all files have been parsed, so a parser can write data that can only be + // determined from the complete set (e.g. dangling references). No-op for parsers that don't + // need it. + void FinalizeDatabase(); + public bool Verbose { get; set; } public bool SkipReferences { get; set; } public bool SkipCrc { get; set; } diff --git a/Analyzer/SQLite/Parsers/AddressablesBuildLayoutParser.cs b/Analyzer/SQLite/Parsers/AddressablesBuildLayoutParser.cs index 5b1bd45..1b3fae7 100644 --- a/Analyzer/SQLite/Parsers/AddressablesBuildLayoutParser.cs +++ b/Analyzer/SQLite/Parsers/AddressablesBuildLayoutParser.cs @@ -4,8 +4,8 @@ using Newtonsoft.Json; using Newtonsoft.Json.Linq; using UnityDataTools.Analyzer.SQLite.Handlers; -using UnityDataTools.Models; using UnityDataTools.Analyzer.SQLite.Writers; +using UnityDataTools.Models; namespace UnityDataTools.Analyzer.SQLite.Parsers { @@ -21,6 +21,12 @@ public void Dispose() { m_Writer.Dispose(); } + + public void FinalizeDatabase() + { + // Addressables build reports don't produce dangling object references. + } + public void Init(SqliteConnection db) { m_Writer = new AddressablesBuildLayoutSQLWriter(db); diff --git a/Analyzer/SQLite/Parsers/SerializedFileParser.cs b/Analyzer/SQLite/Parsers/SerializedFileParser.cs index 2b1993d..ec66c8b 100644 --- a/Analyzer/SQLite/Parsers/SerializedFileParser.cs +++ b/Analyzer/SQLite/Parsers/SerializedFileParser.cs @@ -35,6 +35,12 @@ public void Dispose() m_Writer.Dispose(); } + public void FinalizeDatabase() + { + // m_Writer is only Init'd once a file is actually parsed; nothing to finalize otherwise. + m_Writer.FinalizeDatabase(); + } + public void Init(SqliteConnection db) { m_Writer = new SerializedFileSQLiteWriter(db, SkipReferences, SkipCrc); diff --git a/Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs b/Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs index d36c2d5..e5bb82d 100644 --- a/Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs +++ b/Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs @@ -74,6 +74,7 @@ public class SerializedFileSQLiteWriter : IDisposable private AddObject m_AddObjectCommand = new AddObject(); private AddType m_AddTypeCommand = new AddType(); private AddPreloadDependency m_InsertDepCommand = new AddPreloadDependency(); + private AddDanglingRef m_AddDanglingRefCommand = new AddDanglingRef(); private bool m_Initialized; private SqliteConnection m_Database; @@ -112,6 +113,7 @@ private void CreateSQLiteCommands() m_AddObjectCommand.CreateCommand(m_Database); m_AddTypeCommand.CreateCommand(m_Database); m_InsertDepCommand.CreateCommand(m_Database); + m_AddDanglingRefCommand.CreateCommand(m_Database); m_LastId = m_Database.CreateCommand(); m_LastId.CommandText = "SELECT last_insert_rowid()"; @@ -277,17 +279,22 @@ public void WriteSerializedFile(string relativePath, string fullPath, string con name = randomAccessReader["m_Name"].GetValue(); } + // Resolve m_GameObject to its analyzer object id for the game_object column, but only + // when the PPtr is non-null. A null PPtr (m_FileID 0 and m_PathID 0) is expected for + // any object that is not a component; resolving it would allocate a phantom id for a + // (file, 0) object that never exists and then surface as a bogus dangling ref. + object gameObject = ""; if (randomAccessReader.HasChild("m_GameObject")) { var pptr = randomAccessReader["m_GameObject"]; - var fileId = m_LocalToDbFileId[pptr["m_FileID"].GetValue()]; - var gameObjectID = m_ObjectIdProvider.GetId((fileId, pptr["m_PathID"].GetValue())); - m_AddObjectCommand.SetValue("game_object", gameObjectID); - } - else - { - m_AddObjectCommand.SetValue("game_object", ""); + var gameObjectFileId = pptr["m_FileID"].GetValue(); + var gameObjectPathId = pptr["m_PathID"].GetValue(); + if (gameObjectFileId != 0 || gameObjectPathId != 0) + { + gameObject = m_ObjectIdProvider.GetId((m_LocalToDbFileId[gameObjectFileId], gameObjectPathId)); + } } + m_AddObjectCommand.SetValue("game_object", gameObject); // The walk both extracts references and accumulates the CRC, so it is needed // unless both are disabled. When CRC is on but references are off, the walk @@ -328,6 +335,78 @@ public void WriteSerializedFile(string relativePath, string fullPath, string con } } + // Records every referenced object that was assigned an id but never written to the objects + // table (its serialized file was not part of the analyzed input). Must run after all files are + // processed, since an id is only known to be dangling once nothing has claimed it as an object. + // The set of written objects/files is read back from the database so this is robust to every + // object write site (regular objects, synthetic Scene objects, etc.) without instrumenting them. + public void FinalizeDatabase() + { + // Dangling refs are part of reference tracking, so honor --skip-references (the view that + // makes them useful joins the refs table, which is empty in that mode anyway). + if (!m_Initialized || m_SkipReferences) + return; + + var writtenObjectIds = ReadIntSet("SELECT id FROM objects"); + var writtenFileIds = ReadIntSet("SELECT id FROM serialized_files"); + + // Invert the file-name provider so a dangling target's file id maps back to its name. + var fileIdToName = new Dictionary(); + foreach (var entry in m_SerializedFileIdProvider.Entries) + fileIdToName[entry.Value] = entry.Key; + + using var transaction = m_Database.BeginTransaction(); + try + { + foreach (var entry in m_ObjectIdProvider.Entries) + { + var objectId = entry.Value; + if (writtenObjectIds.Contains(objectId)) + continue; + + var (fileId, pathId) = entry.Key; + + // Ensure the (un-analyzed) target file has a serialized_files row so the dangling + // ref can name it. We have to put null for archive - even if the target file is inside an AssetBundle + // or other archive file, because we simply don't have that information. This is fundamental to + // how references work in Unity. + if (writtenFileIds.Add(fileId)) + { + m_AddSerializedFileCommand.SetTransaction(transaction); + m_AddSerializedFileCommand.SetValue("id", fileId); + m_AddSerializedFileCommand.SetValue("archive", null); + m_AddSerializedFileCommand.SetValue("name", + fileIdToName.TryGetValue(fileId, out var name) ? name : ""); + m_AddSerializedFileCommand.ExecuteNonQuery(); + } + + m_AddDanglingRefCommand.SetTransaction(transaction); + m_AddDanglingRefCommand.SetValue("id", objectId); + m_AddDanglingRefCommand.SetValue("object_id", pathId); + m_AddDanglingRefCommand.SetValue("serialized_file", fileId); + m_AddDanglingRefCommand.ExecuteNonQuery(); + } + + transaction.Commit(); + } + catch (Exception) + { + transaction.Rollback(); + throw; + } + } + + private HashSet ReadIntSet(string sql) + { + var result = new HashSet(); + using var command = m_Database.CreateCommand(); + command.CommandText = sql; + using var reader = command.ExecuteReader(); + while (reader.Read()) + result.Add(reader.GetInt32(0)); + return result; + } + // Callback from PPtrAndCrcProcessor for each reference discovered in the SerializedFile private int AddReference(long objectId, int fileId, long pathId, string propertyPath, string propertyType) { @@ -395,6 +474,7 @@ public void Dispose() m_AddObjectCommand.Dispose(); m_AddTypeCommand.Dispose(); m_InsertDepCommand.Dispose(); + m_AddDanglingRefCommand.Dispose(); m_LastId.Dispose(); } diff --git a/Analyzer/Util/IdProvider.cs b/Analyzer/Util/IdProvider.cs index 20f2732..c587cbf 100644 --- a/Analyzer/Util/IdProvider.cs +++ b/Analyzer/Util/IdProvider.cs @@ -11,6 +11,10 @@ public class IdProvider { private Dictionary m_Ids = new(); + // Exposes the key->id assignments so callers can iterate or invert the mapping (used at + // finalize to map a dangling object id back to its (fileId, pathId) or file name). + public IReadOnlyDictionary Entries => m_Ids; + public int GetId(Key key) { if (m_Ids.TryGetValue(key, out var id)) diff --git a/Documentation/analyzer.md b/Documentation/analyzer.md index 271e94c..52fb897 100644 --- a/Documentation/analyzer.md +++ b/Documentation/analyzer.md @@ -98,7 +98,8 @@ What the `object` is depends on the build: `sharedassetsN.assets`) plus one in `globalgamemanagers.assets` for the always-loaded set. The `dependency` side can reference an object that analyze never recorded, leaving a "dangling" id -with no matching `objects` row. The most common case is objects in `unity default resources`, the +with no matching `objects` row (such ids are catalogued in the `dangling_refs` table). The most +common case is objects in `unity default resources`, the built-in resource file that ships with the Unity Editor without TypeTrees and so cannot be analyzed without a specially built copy. (`Resources/unity_builtin_extra` is built alongside your content and can be analyzed, but produces the same dangling references when it is not part of the analyzed set.) @@ -251,6 +252,36 @@ SELECT * FROM refs_view WHERE property_type = 'MonoScript'; These tables are not populated when analyze is run with `--skip-references`. +## dangling_refs / dangling_refs_view + +When a reference points to an object whose serialized file was not part of the analyzed input, +analyze still assigns that target an object id but never writes an `objects` row for it. `dangling_refs` +records those targets so the reference information is preserved and every object id is accounted for +(each id resolves to exactly one of `objects` or `dangling_refs`, never both). Common causes: + +* Analyzing a single AssetBundle or a partial subset of a bundle group, so cross-bundle references + point at bundles that were not analyzed. +* A Player build referencing `unity default resources` (and `Resources/unity_builtin_extra` when it + is not part of the analyzed set) - built-in files that ship without TypeTrees and are not analyzed. +* A ContentDirectory build analyzed without its `ContentLayout.json`, so references cannot be resolved. + +The columns mirror the `objects` table: `id` (the assigned object id, absent from `objects`), +`object_id` (the target's local file id / LFID) and `serialized_file`. The target file is recorded in +`serialized_files` (with `archive` NULL and no objects of its own) so its name is available; the +lowercased file name is all that is known about it - there is no type, size, or other detail. + +`dangling_refs_view` joins each dangling target back to the object(s) that reference it, one row per +reference, with columns `source_id`, `source_serialized_file`, `source_object_id`, `property_path`, +`property_type`, `target_id`, `target_serialized_file`, `target_object_id`. + +```sql +-- what does object 42 fail to resolve, and where should those objects have come from? +SELECT * FROM dangling_refs_view WHERE source_id = 42; +``` + +Because the view joins `refs`, it is not populated when analyze is run with `--skip-references` +(in that mode neither `refs` nor `dangling_refs` are populated). + ## BuildReport See [BuildReport.md](buildreport.md) for details of the tables and views related to analyzing BuildReport files. diff --git a/UnityDataTool.Tests/AnalyzeDanglingRefsTests.cs b/UnityDataTool.Tests/AnalyzeDanglingRefsTests.cs new file mode 100644 index 0000000..b1c6158 --- /dev/null +++ b/UnityDataTool.Tests/AnalyzeDanglingRefsTests.cs @@ -0,0 +1,99 @@ +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Data.Sqlite; +using NUnit.Framework; + +namespace UnityDataTools.UnityDataTool.Tests; + +#pragma warning disable NUnit2005, NUnit2006 + +// Tests the dangling_refs table/view (issue #85): references to objects whose serialized file was +// not part of the analyzed input get recorded instead of leaving unexplained gaps in the object id +// space. Uses the LeadingEdge AssetBundles fixture, where assetbundleroot.manifest declares +// dependencies on three other bundles, so analyzing assetbundleroot alone leaves cross-bundle +// references dangling; analyzing the whole set resolves them. +public class AnalyzeDanglingRefsTests +{ + private string m_TestOutputFolder; + private string m_AssetBundlesFolder; + + [OneTimeSetUp] + public void OneTimeSetup() + { + m_TestOutputFolder = Path.Combine(TestContext.CurrentContext.TestDirectory, "dangling_refs_test_folder"); + m_AssetBundlesFolder = Path.Combine(TestContext.CurrentContext.TestDirectory, + "Data", "LeadingEdgeBuilds", "AssetBundles"); + Directory.CreateDirectory(m_TestOutputFolder); + Directory.SetCurrentDirectory(m_TestOutputFolder); + } + + [TearDown] + public void Teardown() + { + SqliteConnection.ClearAllPools(); + new DirectoryInfo(m_TestOutputFolder).EnumerateFiles().ToList().ForEach(f => f.Delete()); + } + + // Every referenced object must resolve to exactly one of objects/dangling_refs: nothing left + // unaccounted for, and no id in both tables. This is the core guarantee of the feature. + private static void AssertReferencesFullyAccounted(SqliteConnection db) + { + SQLTestHelper.AssertQueryInt(db, + @"SELECT COUNT(*) FROM refs r + LEFT JOIN objects o ON o.id = r.referenced_object + LEFT JOIN dangling_refs d ON d.id = r.referenced_object + WHERE o.id IS NULL AND d.id IS NULL", + 0, "every refs.referenced_object should resolve to an objects or dangling_refs row"); + SQLTestHelper.AssertQueryInt(db, + "SELECT COUNT(*) FROM dangling_refs d INNER JOIN objects o ON o.id = d.id", + 0, "an id must not be in both objects and dangling_refs"); + // LFID 0 is never a real object; a dangling row with object_id 0 means a null PPtr (e.g. a + // null m_GameObject) was mistakenly resolved to a phantom (file, 0) id. + SQLTestHelper.AssertQueryInt(db, + "SELECT COUNT(*) FROM dangling_refs WHERE object_id = 0", + 0, "dangling_refs should not contain phantom object_id 0 entries"); + } + + [Test] + public async Task Analyze_PartialAssetBundleSet_RecordsCrossBundleDanglingRefs() + { + // Analyze only assetbundleroot; its references into the dependency bundles dangle. + var bundlePath = Path.Combine(m_AssetBundlesFolder, "assetbundleroot"); + var databasePath = SQLTestHelper.GetDatabasePath(m_TestOutputFolder); + + Assert.AreEqual(0, await Program.Main(new string[] { "analyze", bundlePath, "-o", databasePath })); + using var db = SQLTestHelper.OpenDatabase(databasePath); + + Assert.Greater(SQLTestHelper.QueryInt(db, "SELECT COUNT(*) FROM dangling_refs"), 0, + "analyzing a single bundle should leave cross-bundle references dangling"); + Assert.Greater(SQLTestHelper.QueryInt(db, "SELECT COUNT(*) FROM dangling_refs_view"), 0, + "the dangling references should be visible in dangling_refs_view"); + + // The dangling targets live in files that were referenced but not analyzed, so those files + // get a serialized_files row with no objects of their own. + SQLTestHelper.AssertQueryInt(db, + @"SELECT COUNT(*) FROM dangling_refs d + WHERE d.serialized_file IN (SELECT serialized_file FROM objects) + AND (SELECT COUNT(*) FROM refs r WHERE r.referenced_object = d.id) > 0", + 0, "referenced dangling targets should be in files that were not analyzed"); + + AssertReferencesFullyAccounted(db); + } + + [Test] + public async Task Analyze_FullAssetBundleSet_ResolvesCrossBundleReferences() + { + // Analyzing the whole set brings the dependency bundles in, so no reference dangles into an + // un-analyzed file: dangling_refs_view (which joins refs) is empty. + var databasePath = SQLTestHelper.GetDatabasePath(m_TestOutputFolder); + + Assert.AreEqual(0, await Program.Main(new string[] { "analyze", m_AssetBundlesFolder, "-o", databasePath })); + using var db = SQLTestHelper.OpenDatabase(databasePath); + + SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM dangling_refs_view", 0, + "analyzing the full set should resolve every cross-bundle reference"); + + AssertReferencesFullyAccounted(db); + } +} diff --git a/UnityDataTool.Tests/ExpectedDataGenerator.cs b/UnityDataTool.Tests/ExpectedDataGenerator.cs index c5b5a29..bd5fee4 100644 --- a/UnityDataTool.Tests/ExpectedDataGenerator.cs +++ b/UnityDataTool.Tests/ExpectedDataGenerator.cs @@ -44,7 +44,7 @@ public static void Generate(Context context) (SELECT COUNT(*) FROM monoscripts), (SELECT COUNT(*) FROM objects), (SELECT COUNT(*) FROM refs), - (SELECT COUNT(*) FROM serialized_files), + (SELECT COUNT(*) FROM serialized_files WHERE id IN (SELECT serialized_file FROM objects)), (SELECT COUNT(*) FROM shader_subprograms), (SELECT COUNT(*) FROM shaders), (SELECT COUNT(*) FROM shader_keywords), diff --git a/UnityDataTool.Tests/UnityDataToolAssetBundleTests.cs b/UnityDataTool.Tests/UnityDataToolAssetBundleTests.cs index c11fb3d..18b53e3 100644 --- a/UnityDataTool.Tests/UnityDataToolAssetBundleTests.cs +++ b/UnityDataTool.Tests/UnityDataToolAssetBundleTests.cs @@ -334,7 +334,9 @@ private void ValidateDatabase(string databasePath, bool withRefs) (SELECT COUNT(*) FROM meshes), (SELECT COUNT(*) FROM objects), (SELECT COUNT(*) FROM refs), - (SELECT COUNT(*) FROM serialized_files), + -- Count only analyzed files; serialized_files now also holds referenced-but-not-analyzed + -- files that are dangling-ref targets (issue #85), which are not part of this count. + (SELECT COUNT(*) FROM serialized_files WHERE id IN (SELECT serialized_file FROM objects)), (SELECT COUNT(*) FROM shader_subprograms), (SELECT COUNT(*) FROM shaders), (SELECT COUNT(*) FROM shader_keywords), diff --git a/UnityDataTool.Tests/UnityDataToolPlayerDataTests.cs b/UnityDataTool.Tests/UnityDataToolPlayerDataTests.cs index dc65ce5..d399d32 100644 --- a/UnityDataTool.Tests/UnityDataToolPlayerDataTests.cs +++ b/UnityDataTool.Tests/UnityDataToolPlayerDataTests.cs @@ -1,8 +1,8 @@ using System; -using Microsoft.Data.Sqlite; using System.IO; using System.Text.RegularExpressions; using System.Threading.Tasks; +using Microsoft.Data.Sqlite; using NUnit.Framework; using UnityDataTools.TestCommon; @@ -54,7 +54,8 @@ public async Task Analyze_PlayerData_DatabaseCorrect() (SELECT COUNT(*) FROM assetbundle_assets), (SELECT COUNT(*) FROM objects), (SELECT COUNT(*) FROM refs), - (SELECT COUNT(*) FROM serialized_files)"; + -- Analyzed files only; serialized_files also holds dangling-ref target files (issue #85). + (SELECT COUNT(*) FROM serialized_files WHERE id IN (SELECT serialized_file FROM objects))"; using var reader = cmd.ExecuteReader(); @@ -95,6 +96,49 @@ public async Task Analyze_PlayerDataPreloadDependencies_ObjectResolvesToRealObje 0, "every preload_dependencies.object should resolve to an objects row"); } + [Test] + public async Task Analyze_PlayerData_RecordsDanglingRefsToDefaultResources() + { + // Issue #85: a player build references objects in "unity default resources" (shipped without + // TypeTrees, never analyzed). Those references must be recorded in dangling_refs instead of + // leaving unexplained gaps in the object id space. + var analyzePath = Path.Combine(TestContext.CurrentContext.TestDirectory, "Data", "PlayerWithTypeTrees"); + var databasePath = SQLTestHelper.GetDatabasePath(m_TestOutputFolder); + + Assert.AreEqual(0, await Program.Main(new string[] { "analyze", analyzePath, "-o", databasePath })); + using var db = SQLTestHelper.OpenDatabase(databasePath); + + Assert.Greater(SQLTestHelper.QueryInt(db, "SELECT COUNT(*) FROM dangling_refs"), 0, + "the player build should have dangling references (e.g. into unity default resources)"); + + // "unity default resources" is recorded as a serialized_files row and is a dangling target. + Assert.Greater(SQLTestHelper.QueryInt(db, + @"SELECT COUNT(*) FROM dangling_refs d + INNER JOIN serialized_files sf ON sf.id = d.serialized_file + WHERE sf.name = 'unity default resources'"), + 0, "expected dangling references into 'unity default resources'"); + + // The dangling references are visible in the view, with real analyzed source files. + Assert.Greater(SQLTestHelper.QueryInt(db, "SELECT COUNT(*) FROM dangling_refs_view"), 0, + "dangling references should be visible in dangling_refs_view"); + + // Every referenced object resolves to exactly one of objects/dangling_refs. + SQLTestHelper.AssertQueryInt(db, + @"SELECT COUNT(*) FROM refs r + LEFT JOIN objects o ON o.id = r.referenced_object + LEFT JOIN dangling_refs d ON d.id = r.referenced_object + WHERE o.id IS NULL AND d.id IS NULL", + 0, "every refs.referenced_object should resolve to an objects or dangling_refs row"); + SQLTestHelper.AssertQueryInt(db, + "SELECT COUNT(*) FROM dangling_refs d INNER JOIN objects o ON o.id = d.id", + 0, "an id must not be in both objects and dangling_refs"); + // LFID 0 is never a real object; a dangling row with object_id 0 means a null PPtr (e.g. a + // null m_GameObject) was mistakenly resolved to a phantom (file, 0) id. + SQLTestHelper.AssertQueryInt(db, + "SELECT COUNT(*) FROM dangling_refs WHERE object_id = 0", + 0, "dangling_refs should not contain phantom object_id 0 entries"); + } + [Test] public async Task DumpText_PlayerData_TextFileCreatedCorrectly() {