From d98235e1533a50e27ce5a3964ba3ae3e63e17640 Mon Sep 17 00:00:00 2001 From: Andrew Skowronski Date: Tue, 14 Jul 2026 10:06:27 -0400 Subject: [PATCH 1/4] [#85] Track dangling references in analyze Add a dangling_refs table and dangling_refs_view recording references whose target object was assigned an id but never written to the objects table because its serialized file was not part of the analyzed input (partial bundle sets, unity default resources, ContentDirectory builds without ContentLayout.json). Every object id now resolves to exactly one of objects or dangling_refs. Detection runs after all files are processed via a new ISQLiteFileParser.FinalizeDatabase() hook: the writer reads back the written object/file ids and records any assigned id with no objects row, adding a serialized_files row (archive NULL) for each un-analyzed target file so its name is available. Honors --skip-references. Schema user_version bumped 4 -> 5. Existing serialized_files count assertions now count analyzed files only (equal to the previous golden values, so no ExpectedData regeneration). --- Analyzer/AnalyzerTool.cs | 9 +- Analyzer/Resources/Init.sql | 47 +++++++++- .../Commands/SerializedFile/AddDanglingRef.cs | 29 ++++++ Analyzer/SQLite/Handlers/ISQLiteHandler.cs | 6 ++ .../Parsers/AddressablesBuildLayoutParser.cs | 8 +- .../SQLite/Parsers/SerializedFileParser.cs | 6 ++ .../Writers/SerializedFileSQLiteWriter.cs | 73 ++++++++++++++ Analyzer/Util/IdProvider.cs | 4 + Documentation/analyzer.md | 34 ++++++- .../AnalyzeDanglingRefsTests.cs | 94 +++++++++++++++++++ UnityDataTool.Tests/ExpectedDataGenerator.cs | 2 +- .../UnityDataToolAssetBundleTests.cs | 4 +- .../UnityDataToolPlayerDataTests.cs | 43 ++++++++- 13 files changed, 350 insertions(+), 9 deletions(-) create mode 100644 Analyzer/SQLite/Commands/SerializedFile/AddDanglingRef.cs create mode 100644 UnityDataTool.Tests/AnalyzeDanglingRefsTests.cs diff --git a/Analyzer/AnalyzerTool.cs b/Analyzer/AnalyzerTool.cs index 918ef96..da8473c 100644 --- a/Analyzer/AnalyzerTool.cs +++ b/Analyzer/AnalyzerTool.cs @@ -5,8 +5,8 @@ using UnityDataTools.Analyzer.SQLite.Handlers; using UnityDataTools.Analyzer.SQLite.Parsers; using UnityDataTools.Analyzer.SQLite.Writers; -using UnityDataTools.Models; using UnityDataTools.FileSystem; +using UnityDataTools.Models; namespace UnityDataTools.Analyzer; @@ -131,6 +131,13 @@ public int Analyze(AnalyzeOptions options) Console.WriteLine(); Console.WriteLine($"Finalizing database. Successfully processed files: {countSuccess}, Failed files: {countFailures}, Files without TypeTrees: {countNoTypeTrees}, Ignored files: {countIgnored}"); + // Record data that can only be determined once every file has been processed (e.g. which + // referenced objects were never resolved) before the database is finalized. + foreach (var parser in parsers) + { + parser.FinalizeDatabase(); + } + writer.End(); foreach (var parser in parsers) { diff --git a/Analyzer/Resources/Init.sql b/Analyzer/Resources/Init.sql index 8e9480f..ac5159b 100644 --- a/Analyzer/Resources/Init.sql +++ b/Analyzer/Resources/Init.sql @@ -81,6 +81,25 @@ CREATE TABLE IF NOT EXISTS refs property_type INTEGER ); +-- One row per referenced object that analyze assigned an id to but never wrote an objects row +-- for (its serialized file was not part of the analyzed input). Written after all files are +-- processed, once we can tell which assigned ids never became objects. Common causes: analyzing a +-- partial set of AssetBundles, references into "unity default resources" (shipped without +-- TypeTrees), or a ContentDirectory build analyzed without ContentLayout.json. +-- Columns mirror the objects table so every object id resolves to exactly one of objects or +-- dangling_refs: +-- id - the analyzer object id (no row in objects has this id). +-- object_id - the target's local file id (LFID / m_PathID) within its serialized file. +-- serialized_file - references the serialized_files row of the (un-analyzed) file the target +-- lives in. That row has archive NULL and no objects of its own. +CREATE TABLE IF NOT EXISTS dangling_refs +( + id INTEGER, + object_id INTEGER, + serialized_file INTEGER, + PRIMARY KEY (id) +); + -- Resolves the property_path and property_type ids in the refs table to their string values. CREATE VIEW refs_view AS SELECT r.object, r.referenced_object, pn.name AS property_path, pt.name AS property_type @@ -88,6 +107,29 @@ FROM refs r INNER JOIN property_names pn ON r.property_path = pn.id INNER JOIN property_types pt ON r.property_type = pt.id; +-- Resolves dangling_refs to the source object(s) that reference each missing target, one row per +-- (referencing object -> dangling target) reference. Because it joins refs, this view is empty +-- when analyze is run with --skip-references even though the dangling_refs table may be populated. +-- A dangling target only reached through preload_dependencies / game_object (not a refs row) is in +-- the table but not here. +CREATE VIEW dangling_refs_view AS +SELECT + r.object AS source_id, + src_sf.name AS source_serialized_file, + src_o.object_id AS source_object_id, + pn.name AS property_path, + pt.name AS property_type, + d.id AS target_id, + tgt_sf.name AS target_serialized_file, + d.object_id AS target_object_id +FROM dangling_refs d +INNER JOIN refs r ON r.referenced_object = d.id +INNER JOIN objects src_o ON r.object = src_o.id +INNER JOIN serialized_files src_sf ON src_o.serialized_file = src_sf.id +INNER JOIN serialized_files tgt_sf ON d.serialized_file = tgt_sf.id +LEFT JOIN property_names pn ON r.property_path = pn.id +LEFT JOIN property_types pt ON r.property_type = pt.id; + CREATE VIEW object_view AS SELECT o.id, o.object_id, ab.name AS archive, sf.name AS serialized_file, t.name AS type, o.name, o.game_object, o.size, CASE @@ -157,8 +199,9 @@ INSERT INTO types (id, name) VALUES (-1, 'Scene'); -- 1 = normalized refs table (issue #44); 2 = renamed assets/asset_dependencies tables to -- assetbundle_assets/preload_dependencies (issue #82); 3 = renamed asset_bundles table to archives -- and the asset_bundle column/alias to archive (issue #68); 4 = build_report_packed_asset_contents_view --- type column changed from numeric id to type name (issue #55); databases produced before versioning report 0. -PRAGMA user_version = 4; +-- type column changed from numeric id to type name (issue #55); 5 = added dangling_refs table/view +-- (issue #85); databases produced before versioning report 0. +PRAGMA user_version = 5; PRAGMA synchronous = OFF; PRAGMA journal_mode = MEMORY; diff --git a/Analyzer/SQLite/Commands/SerializedFile/AddDanglingRef.cs b/Analyzer/SQLite/Commands/SerializedFile/AddDanglingRef.cs new file mode 100644 index 0000000..1af0907 --- /dev/null +++ b/Analyzer/SQLite/Commands/SerializedFile/AddDanglingRef.cs @@ -0,0 +1,29 @@ +using System.Collections.Generic; +using Microsoft.Data.Sqlite; +using UnityDataTools.Analyzer.SQLite.Commands; + +namespace UnityDataTools.Analyzer.SQLite.Commands.SerializedFile +{ + /* TABLE DEFINITION: + create table dangling_refs + ( + id INTEGER, + object_id INTEGER, + serialized_file INTEGER, + PRIMARY KEY (id) + ); + */ + internal class AddDanglingRef : AbstractCommand + { + protected override string TableName => "dangling_refs"; + + protected override string DDLSource => null; + + protected override Dictionary Fields => new() + { + { "id", SqliteType.Integer }, + { "object_id", SqliteType.Integer }, + { "serialized_file", SqliteType.Integer } + }; + } +} diff --git a/Analyzer/SQLite/Handlers/ISQLiteHandler.cs b/Analyzer/SQLite/Handlers/ISQLiteHandler.cs index c5f8806..d266272 100644 --- a/Analyzer/SQLite/Handlers/ISQLiteHandler.cs +++ b/Analyzer/SQLite/Handlers/ISQLiteHandler.cs @@ -27,6 +27,12 @@ public interface ISQLiteFileParser : IDisposable void Init(SqliteConnection db); bool CanParse(string filename); void Parse(string filename); + + // Called once after all files have been parsed, so a parser can write data that can only be + // determined from the complete set (e.g. dangling references). No-op for parsers that don't + // need it. + void FinalizeDatabase(); + public bool Verbose { get; set; } public bool SkipReferences { get; set; } public bool SkipCrc { get; set; } diff --git a/Analyzer/SQLite/Parsers/AddressablesBuildLayoutParser.cs b/Analyzer/SQLite/Parsers/AddressablesBuildLayoutParser.cs index 5b1bd45..1b3fae7 100644 --- a/Analyzer/SQLite/Parsers/AddressablesBuildLayoutParser.cs +++ b/Analyzer/SQLite/Parsers/AddressablesBuildLayoutParser.cs @@ -4,8 +4,8 @@ using Newtonsoft.Json; using Newtonsoft.Json.Linq; using UnityDataTools.Analyzer.SQLite.Handlers; -using UnityDataTools.Models; using UnityDataTools.Analyzer.SQLite.Writers; +using UnityDataTools.Models; namespace UnityDataTools.Analyzer.SQLite.Parsers { @@ -21,6 +21,12 @@ public void Dispose() { m_Writer.Dispose(); } + + public void FinalizeDatabase() + { + // Addressables build reports don't produce dangling object references. + } + public void Init(SqliteConnection db) { m_Writer = new AddressablesBuildLayoutSQLWriter(db); diff --git a/Analyzer/SQLite/Parsers/SerializedFileParser.cs b/Analyzer/SQLite/Parsers/SerializedFileParser.cs index 2b1993d..ec66c8b 100644 --- a/Analyzer/SQLite/Parsers/SerializedFileParser.cs +++ b/Analyzer/SQLite/Parsers/SerializedFileParser.cs @@ -35,6 +35,12 @@ public void Dispose() m_Writer.Dispose(); } + public void FinalizeDatabase() + { + // m_Writer is only Init'd once a file is actually parsed; nothing to finalize otherwise. + m_Writer.FinalizeDatabase(); + } + public void Init(SqliteConnection db) { m_Writer = new SerializedFileSQLiteWriter(db, SkipReferences, SkipCrc); diff --git a/Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs b/Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs index d36c2d5..b443361 100644 --- a/Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs +++ b/Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs @@ -74,6 +74,7 @@ public class SerializedFileSQLiteWriter : IDisposable private AddObject m_AddObjectCommand = new AddObject(); private AddType m_AddTypeCommand = new AddType(); private AddPreloadDependency m_InsertDepCommand = new AddPreloadDependency(); + private AddDanglingRef m_AddDanglingRefCommand = new AddDanglingRef(); private bool m_Initialized; private SqliteConnection m_Database; @@ -112,6 +113,7 @@ private void CreateSQLiteCommands() m_AddObjectCommand.CreateCommand(m_Database); m_AddTypeCommand.CreateCommand(m_Database); m_InsertDepCommand.CreateCommand(m_Database); + m_AddDanglingRefCommand.CreateCommand(m_Database); m_LastId = m_Database.CreateCommand(); m_LastId.CommandText = "SELECT last_insert_rowid()"; @@ -328,6 +330,76 @@ public void WriteSerializedFile(string relativePath, string fullPath, string con } } + // Records every referenced object that was assigned an id but never written to the objects + // table (its serialized file was not part of the analyzed input). Must run after all files are + // processed, since an id is only known to be dangling once nothing has claimed it as an object. + // The set of written objects/files is read back from the database so this is robust to every + // object write site (regular objects, synthetic Scene objects, etc.) without instrumenting them. + public void FinalizeDatabase() + { + // Dangling refs are part of reference tracking, so honor --skip-references (the view that + // makes them useful joins the refs table, which is empty in that mode anyway). + if (!m_Initialized || m_SkipReferences) + return; + + var writtenObjectIds = ReadIntSet("SELECT id FROM objects"); + var writtenFileIds = ReadIntSet("SELECT id FROM serialized_files"); + + // Invert the file-name provider so a dangling target's file id maps back to its name. + var fileIdToName = new Dictionary(); + foreach (var entry in m_SerializedFileIdProvider.Entries) + fileIdToName[entry.Value] = entry.Key; + + using var transaction = m_Database.BeginTransaction(); + try + { + foreach (var entry in m_ObjectIdProvider.Entries) + { + var objectId = entry.Value; + if (writtenObjectIds.Contains(objectId)) + continue; + + var (fileId, pathId) = entry.Key; + + // Ensure the (un-analyzed) target file has a serialized_files row so the dangling + // ref can name it. archive is NULL: the file was referenced, not opened. + if (writtenFileIds.Add(fileId)) + { + m_AddSerializedFileCommand.SetTransaction(transaction); + m_AddSerializedFileCommand.SetValue("id", fileId); + m_AddSerializedFileCommand.SetValue("archive", null); + m_AddSerializedFileCommand.SetValue("name", + fileIdToName.TryGetValue(fileId, out var name) ? name : ""); + m_AddSerializedFileCommand.ExecuteNonQuery(); + } + + m_AddDanglingRefCommand.SetTransaction(transaction); + m_AddDanglingRefCommand.SetValue("id", objectId); + m_AddDanglingRefCommand.SetValue("object_id", pathId); + m_AddDanglingRefCommand.SetValue("serialized_file", fileId); + m_AddDanglingRefCommand.ExecuteNonQuery(); + } + + transaction.Commit(); + } + catch (Exception) + { + transaction.Rollback(); + throw; + } + } + + private HashSet ReadIntSet(string sql) + { + var result = new HashSet(); + using var command = m_Database.CreateCommand(); + command.CommandText = sql; + using var reader = command.ExecuteReader(); + while (reader.Read()) + result.Add(reader.GetInt32(0)); + return result; + } + // Callback from PPtrAndCrcProcessor for each reference discovered in the SerializedFile private int AddReference(long objectId, int fileId, long pathId, string propertyPath, string propertyType) { @@ -395,6 +467,7 @@ public void Dispose() m_AddObjectCommand.Dispose(); m_AddTypeCommand.Dispose(); m_InsertDepCommand.Dispose(); + m_AddDanglingRefCommand.Dispose(); m_LastId.Dispose(); } diff --git a/Analyzer/Util/IdProvider.cs b/Analyzer/Util/IdProvider.cs index 20f2732..c587cbf 100644 --- a/Analyzer/Util/IdProvider.cs +++ b/Analyzer/Util/IdProvider.cs @@ -11,6 +11,10 @@ public class IdProvider { private Dictionary m_Ids = new(); + // Exposes the key->id assignments so callers can iterate or invert the mapping (used at + // finalize to map a dangling object id back to its (fileId, pathId) or file name). + public IReadOnlyDictionary Entries => m_Ids; + public int GetId(Key key) { if (m_Ids.TryGetValue(key, out var id)) diff --git a/Documentation/analyzer.md b/Documentation/analyzer.md index 271e94c..e37373e 100644 --- a/Documentation/analyzer.md +++ b/Documentation/analyzer.md @@ -98,7 +98,8 @@ What the `object` is depends on the build: `sharedassetsN.assets`) plus one in `globalgamemanagers.assets` for the always-loaded set. The `dependency` side can reference an object that analyze never recorded, leaving a "dangling" id -with no matching `objects` row. The most common case is objects in `unity default resources`, the +with no matching `objects` row (such ids are catalogued in the `dangling_refs` table). The most +common case is objects in `unity default resources`, the built-in resource file that ships with the Unity Editor without TypeTrees and so cannot be analyzed without a specially built copy. (`Resources/unity_builtin_extra` is built alongside your content and can be analyzed, but produces the same dangling references when it is not part of the analyzed set.) @@ -251,6 +252,37 @@ SELECT * FROM refs_view WHERE property_type = 'MonoScript'; These tables are not populated when analyze is run with `--skip-references`. +## dangling_refs / dangling_refs_view + +When a reference points to an object whose serialized file was not part of the analyzed input, +analyze still assigns that target an object id but never writes an `objects` row for it. `dangling_refs` +records those targets so the reference information is preserved and every object id is accounted for +(each id resolves to exactly one of `objects` or `dangling_refs`, never both). Common causes: + +* Analyzing a single AssetBundle or a partial subset of a bundle group, so cross-bundle references + point at bundles that were not analyzed. +* A Player build referencing `unity default resources` (and `Resources/unity_builtin_extra` when it + is not part of the analyzed set) - built-in files that ship without TypeTrees and are not analyzed. +* A ContentDirectory build analyzed without its `ContentLayout.json`, so references cannot be resolved. + +The columns mirror the `objects` table: `id` (the assigned object id, absent from `objects`), +`object_id` (the target's local file id / LFID) and `serialized_file`. The target file is recorded in +`serialized_files` (with `archive` NULL and no objects of its own) so its name is available; the +lowercased file name is all that is known about it - there is no type, size, or other detail. + +`dangling_refs_view` joins each dangling target back to the object(s) that reference it, one row per +reference, with columns `source_id`, `source_serialized_file`, `source_object_id`, `property_path`, +`property_type`, `target_id`, `target_serialized_file`, `target_object_id`. + +```sql +-- what does object 42 fail to resolve, and where should those objects have come from? +SELECT * FROM dangling_refs_view WHERE source_id = 42; +``` + +Because the view joins `refs`, it is empty when analyze is run with `--skip-references`; in that mode +the `dangling_refs` table is not populated either. A dangling target reached only through +`preload_dependencies` or a `game_object` link (not a `refs` row) appears in the table but not the view. + ## BuildReport See [BuildReport.md](buildreport.md) for details of the tables and views related to analyzing BuildReport files. diff --git a/UnityDataTool.Tests/AnalyzeDanglingRefsTests.cs b/UnityDataTool.Tests/AnalyzeDanglingRefsTests.cs new file mode 100644 index 0000000..9324285 --- /dev/null +++ b/UnityDataTool.Tests/AnalyzeDanglingRefsTests.cs @@ -0,0 +1,94 @@ +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Data.Sqlite; +using NUnit.Framework; + +namespace UnityDataTools.UnityDataTool.Tests; + +#pragma warning disable NUnit2005, NUnit2006 + +// Tests the dangling_refs table/view (issue #85): references to objects whose serialized file was +// not part of the analyzed input get recorded instead of leaving unexplained gaps in the object id +// space. Uses the LeadingEdge AssetBundles fixture, where assetbundleroot.manifest declares +// dependencies on three other bundles, so analyzing assetbundleroot alone leaves cross-bundle +// references dangling; analyzing the whole set resolves them. +public class AnalyzeDanglingRefsTests +{ + private string m_TestOutputFolder; + private string m_AssetBundlesFolder; + + [OneTimeSetUp] + public void OneTimeSetup() + { + m_TestOutputFolder = Path.Combine(TestContext.CurrentContext.TestDirectory, "dangling_refs_test_folder"); + m_AssetBundlesFolder = Path.Combine(TestContext.CurrentContext.TestDirectory, + "Data", "LeadingEdgeBuilds", "AssetBundles"); + Directory.CreateDirectory(m_TestOutputFolder); + Directory.SetCurrentDirectory(m_TestOutputFolder); + } + + [TearDown] + public void Teardown() + { + SqliteConnection.ClearAllPools(); + new DirectoryInfo(m_TestOutputFolder).EnumerateFiles().ToList().ForEach(f => f.Delete()); + } + + // Every referenced object must resolve to exactly one of objects/dangling_refs: nothing left + // unaccounted for, and no id in both tables. This is the core guarantee of the feature. + private static void AssertReferencesFullyAccounted(SqliteConnection db) + { + SQLTestHelper.AssertQueryInt(db, + @"SELECT COUNT(*) FROM refs r + LEFT JOIN objects o ON o.id = r.referenced_object + LEFT JOIN dangling_refs d ON d.id = r.referenced_object + WHERE o.id IS NULL AND d.id IS NULL", + 0, "every refs.referenced_object should resolve to an objects or dangling_refs row"); + SQLTestHelper.AssertQueryInt(db, + "SELECT COUNT(*) FROM dangling_refs d INNER JOIN objects o ON o.id = d.id", + 0, "an id must not be in both objects and dangling_refs"); + } + + [Test] + public async Task Analyze_PartialAssetBundleSet_RecordsCrossBundleDanglingRefs() + { + // Analyze only assetbundleroot; its references into the dependency bundles dangle. + var bundlePath = Path.Combine(m_AssetBundlesFolder, "assetbundleroot"); + var databasePath = SQLTestHelper.GetDatabasePath(m_TestOutputFolder); + + Assert.AreEqual(0, await Program.Main(new string[] { "analyze", bundlePath, "-o", databasePath })); + using var db = SQLTestHelper.OpenDatabase(databasePath); + + Assert.Greater(SQLTestHelper.QueryInt(db, "SELECT COUNT(*) FROM dangling_refs"), 0, + "analyzing a single bundle should leave cross-bundle references dangling"); + Assert.Greater(SQLTestHelper.QueryInt(db, "SELECT COUNT(*) FROM dangling_refs_view"), 0, + "the dangling references should be visible in dangling_refs_view"); + + // The dangling targets live in files that were referenced but not analyzed, so those files + // get a serialized_files row with no objects of their own. + SQLTestHelper.AssertQueryInt(db, + @"SELECT COUNT(*) FROM dangling_refs d + WHERE d.serialized_file IN (SELECT serialized_file FROM objects) + AND (SELECT COUNT(*) FROM refs r WHERE r.referenced_object = d.id) > 0", + 0, "referenced dangling targets should be in files that were not analyzed"); + + AssertReferencesFullyAccounted(db); + } + + [Test] + public async Task Analyze_FullAssetBundleSet_ResolvesCrossBundleReferences() + { + // Analyzing the whole set brings the dependency bundles in, so no reference dangles into an + // un-analyzed file: dangling_refs_view (which joins refs) is empty. + var databasePath = SQLTestHelper.GetDatabasePath(m_TestOutputFolder); + + Assert.AreEqual(0, await Program.Main(new string[] { "analyze", m_AssetBundlesFolder, "-o", databasePath })); + using var db = SQLTestHelper.OpenDatabase(databasePath); + + SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM dangling_refs_view", 0, + "analyzing the full set should resolve every cross-bundle reference"); + + AssertReferencesFullyAccounted(db); + } +} diff --git a/UnityDataTool.Tests/ExpectedDataGenerator.cs b/UnityDataTool.Tests/ExpectedDataGenerator.cs index c5b5a29..bd5fee4 100644 --- a/UnityDataTool.Tests/ExpectedDataGenerator.cs +++ b/UnityDataTool.Tests/ExpectedDataGenerator.cs @@ -44,7 +44,7 @@ public static void Generate(Context context) (SELECT COUNT(*) FROM monoscripts), (SELECT COUNT(*) FROM objects), (SELECT COUNT(*) FROM refs), - (SELECT COUNT(*) FROM serialized_files), + (SELECT COUNT(*) FROM serialized_files WHERE id IN (SELECT serialized_file FROM objects)), (SELECT COUNT(*) FROM shader_subprograms), (SELECT COUNT(*) FROM shaders), (SELECT COUNT(*) FROM shader_keywords), diff --git a/UnityDataTool.Tests/UnityDataToolAssetBundleTests.cs b/UnityDataTool.Tests/UnityDataToolAssetBundleTests.cs index c11fb3d..18b53e3 100644 --- a/UnityDataTool.Tests/UnityDataToolAssetBundleTests.cs +++ b/UnityDataTool.Tests/UnityDataToolAssetBundleTests.cs @@ -334,7 +334,9 @@ private void ValidateDatabase(string databasePath, bool withRefs) (SELECT COUNT(*) FROM meshes), (SELECT COUNT(*) FROM objects), (SELECT COUNT(*) FROM refs), - (SELECT COUNT(*) FROM serialized_files), + -- Count only analyzed files; serialized_files now also holds referenced-but-not-analyzed + -- files that are dangling-ref targets (issue #85), which are not part of this count. + (SELECT COUNT(*) FROM serialized_files WHERE id IN (SELECT serialized_file FROM objects)), (SELECT COUNT(*) FROM shader_subprograms), (SELECT COUNT(*) FROM shaders), (SELECT COUNT(*) FROM shader_keywords), diff --git a/UnityDataTool.Tests/UnityDataToolPlayerDataTests.cs b/UnityDataTool.Tests/UnityDataToolPlayerDataTests.cs index dc65ce5..da56d1f 100644 --- a/UnityDataTool.Tests/UnityDataToolPlayerDataTests.cs +++ b/UnityDataTool.Tests/UnityDataToolPlayerDataTests.cs @@ -1,8 +1,8 @@ using System; -using Microsoft.Data.Sqlite; using System.IO; using System.Text.RegularExpressions; using System.Threading.Tasks; +using Microsoft.Data.Sqlite; using NUnit.Framework; using UnityDataTools.TestCommon; @@ -54,7 +54,8 @@ public async Task Analyze_PlayerData_DatabaseCorrect() (SELECT COUNT(*) FROM assetbundle_assets), (SELECT COUNT(*) FROM objects), (SELECT COUNT(*) FROM refs), - (SELECT COUNT(*) FROM serialized_files)"; + -- Analyzed files only; serialized_files also holds dangling-ref target files (issue #85). + (SELECT COUNT(*) FROM serialized_files WHERE id IN (SELECT serialized_file FROM objects))"; using var reader = cmd.ExecuteReader(); @@ -95,6 +96,44 @@ public async Task Analyze_PlayerDataPreloadDependencies_ObjectResolvesToRealObje 0, "every preload_dependencies.object should resolve to an objects row"); } + [Test] + public async Task Analyze_PlayerData_RecordsDanglingRefsToDefaultResources() + { + // Issue #85: a player build references objects in "unity default resources" (shipped without + // TypeTrees, never analyzed). Those references must be recorded in dangling_refs instead of + // leaving unexplained gaps in the object id space. + var analyzePath = Path.Combine(TestContext.CurrentContext.TestDirectory, "Data", "PlayerWithTypeTrees"); + var databasePath = SQLTestHelper.GetDatabasePath(m_TestOutputFolder); + + Assert.AreEqual(0, await Program.Main(new string[] { "analyze", analyzePath, "-o", databasePath })); + using var db = SQLTestHelper.OpenDatabase(databasePath); + + Assert.Greater(SQLTestHelper.QueryInt(db, "SELECT COUNT(*) FROM dangling_refs"), 0, + "the player build should have dangling references (e.g. into unity default resources)"); + + // "unity default resources" is recorded as a serialized_files row and is a dangling target. + Assert.Greater(SQLTestHelper.QueryInt(db, + @"SELECT COUNT(*) FROM dangling_refs d + INNER JOIN serialized_files sf ON sf.id = d.serialized_file + WHERE sf.name = 'unity default resources'"), + 0, "expected dangling references into 'unity default resources'"); + + // The dangling references are visible in the view, with real analyzed source files. + Assert.Greater(SQLTestHelper.QueryInt(db, "SELECT COUNT(*) FROM dangling_refs_view"), 0, + "dangling references should be visible in dangling_refs_view"); + + // Every referenced object resolves to exactly one of objects/dangling_refs. + SQLTestHelper.AssertQueryInt(db, + @"SELECT COUNT(*) FROM refs r + LEFT JOIN objects o ON o.id = r.referenced_object + LEFT JOIN dangling_refs d ON d.id = r.referenced_object + WHERE o.id IS NULL AND d.id IS NULL", + 0, "every refs.referenced_object should resolve to an objects or dangling_refs row"); + SQLTestHelper.AssertQueryInt(db, + "SELECT COUNT(*) FROM dangling_refs d INNER JOIN objects o ON o.id = d.id", + 0, "an id must not be in both objects and dangling_refs"); + } + [Test] public async Task DumpText_PlayerData_TextFileCreatedCorrectly() { From 4e04fc26b9ff5c199005adb73b60f27a500d8373 Mon Sep 17 00:00:00 2001 From: Andrew Skowronski Date: Tue, 14 Jul 2026 10:14:14 -0400 Subject: [PATCH 2/4] [#85] Don't record null m_GameObject as a dangling ref The writer resolved a component's m_GameObject PPtr to an object id even when it was null (m_FileID 0 and m_PathID 0), allocating a phantom (file, 0) object that no row is ever written for. With dangling_refs tracking this surfaced as confusing object_id 0 entries pointing at already-analyzed files. Only resolve m_GameObject when the PPtr is non-null; otherwise leave the game_object column empty, as the no-m_GameObject branch already does. A complete AssetBundle build now has zero dangling_refs, and every dangling target is reachable through refs, so dangling_refs_view covers them all. Tests assert dangling_refs has no object_id 0 rows. --- Analyzer/Resources/Init.sql | 6 ++---- .../Writers/SerializedFileSQLiteWriter.cs | 19 ++++++++++++------- Documentation/analyzer.md | 5 ++--- .../AnalyzeDanglingRefsTests.cs | 5 +++++ .../UnityDataToolPlayerDataTests.cs | 5 +++++ 5 files changed, 26 insertions(+), 14 deletions(-) diff --git a/Analyzer/Resources/Init.sql b/Analyzer/Resources/Init.sql index ac5159b..7cbde59 100644 --- a/Analyzer/Resources/Init.sql +++ b/Analyzer/Resources/Init.sql @@ -108,10 +108,8 @@ INNER JOIN property_names pn ON r.property_path = pn.id INNER JOIN property_types pt ON r.property_type = pt.id; -- Resolves dangling_refs to the source object(s) that reference each missing target, one row per --- (referencing object -> dangling target) reference. Because it joins refs, this view is empty --- when analyze is run with --skip-references even though the dangling_refs table may be populated. --- A dangling target only reached through preload_dependencies / game_object (not a refs row) is in --- the table but not here. +-- (referencing object -> dangling target) reference. Not populated when analyze is run with +-- --skip-references (neither refs nor dangling_refs are populated in that mode). CREATE VIEW dangling_refs_view AS SELECT r.object AS source_id, diff --git a/Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs b/Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs index b443361..b354c11 100644 --- a/Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs +++ b/Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs @@ -279,17 +279,22 @@ public void WriteSerializedFile(string relativePath, string fullPath, string con name = randomAccessReader["m_Name"].GetValue(); } + // Resolve m_GameObject to its analyzer object id for the game_object column, but only + // when the PPtr is non-null. A null PPtr (m_FileID 0 and m_PathID 0) is expected for + // any object that is not a component; resolving it would allocate a phantom id for a + // (file, 0) object that never exists and then surface as a bogus dangling ref. + object gameObject = ""; if (randomAccessReader.HasChild("m_GameObject")) { var pptr = randomAccessReader["m_GameObject"]; - var fileId = m_LocalToDbFileId[pptr["m_FileID"].GetValue()]; - var gameObjectID = m_ObjectIdProvider.GetId((fileId, pptr["m_PathID"].GetValue())); - m_AddObjectCommand.SetValue("game_object", gameObjectID); - } - else - { - m_AddObjectCommand.SetValue("game_object", ""); + var gameObjectFileId = pptr["m_FileID"].GetValue(); + var gameObjectPathId = pptr["m_PathID"].GetValue(); + if (gameObjectFileId != 0 || gameObjectPathId != 0) + { + gameObject = m_ObjectIdProvider.GetId((m_LocalToDbFileId[gameObjectFileId], gameObjectPathId)); + } } + m_AddObjectCommand.SetValue("game_object", gameObject); // The walk both extracts references and accumulates the CRC, so it is needed // unless both are disabled. When CRC is on but references are off, the walk diff --git a/Documentation/analyzer.md b/Documentation/analyzer.md index e37373e..52fb897 100644 --- a/Documentation/analyzer.md +++ b/Documentation/analyzer.md @@ -279,9 +279,8 @@ reference, with columns `source_id`, `source_serialized_file`, `source_object_id SELECT * FROM dangling_refs_view WHERE source_id = 42; ``` -Because the view joins `refs`, it is empty when analyze is run with `--skip-references`; in that mode -the `dangling_refs` table is not populated either. A dangling target reached only through -`preload_dependencies` or a `game_object` link (not a `refs` row) appears in the table but not the view. +Because the view joins `refs`, it is not populated when analyze is run with `--skip-references` +(in that mode neither `refs` nor `dangling_refs` are populated). ## BuildReport diff --git a/UnityDataTool.Tests/AnalyzeDanglingRefsTests.cs b/UnityDataTool.Tests/AnalyzeDanglingRefsTests.cs index 9324285..b1c6158 100644 --- a/UnityDataTool.Tests/AnalyzeDanglingRefsTests.cs +++ b/UnityDataTool.Tests/AnalyzeDanglingRefsTests.cs @@ -48,6 +48,11 @@ private static void AssertReferencesFullyAccounted(SqliteConnection db) SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM dangling_refs d INNER JOIN objects o ON o.id = d.id", 0, "an id must not be in both objects and dangling_refs"); + // LFID 0 is never a real object; a dangling row with object_id 0 means a null PPtr (e.g. a + // null m_GameObject) was mistakenly resolved to a phantom (file, 0) id. + SQLTestHelper.AssertQueryInt(db, + "SELECT COUNT(*) FROM dangling_refs WHERE object_id = 0", + 0, "dangling_refs should not contain phantom object_id 0 entries"); } [Test] diff --git a/UnityDataTool.Tests/UnityDataToolPlayerDataTests.cs b/UnityDataTool.Tests/UnityDataToolPlayerDataTests.cs index da56d1f..d399d32 100644 --- a/UnityDataTool.Tests/UnityDataToolPlayerDataTests.cs +++ b/UnityDataTool.Tests/UnityDataToolPlayerDataTests.cs @@ -132,6 +132,11 @@ public async Task Analyze_PlayerData_RecordsDanglingRefsToDefaultResources() SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM dangling_refs d INNER JOIN objects o ON o.id = d.id", 0, "an id must not be in both objects and dangling_refs"); + // LFID 0 is never a real object; a dangling row with object_id 0 means a null PPtr (e.g. a + // null m_GameObject) was mistakenly resolved to a phantom (file, 0) id. + SQLTestHelper.AssertQueryInt(db, + "SELECT COUNT(*) FROM dangling_refs WHERE object_id = 0", + 0, "dangling_refs should not contain phantom object_id 0 entries"); } [Test] From 47ac39fa8153c28d691279064b08d057f6b1d0d7 Mon Sep 17 00:00:00 2001 From: Andrew Skowronski Date: Tue, 14 Jul 2026 10:53:49 -0400 Subject: [PATCH 3/4] [#85] Skip null-PPtr AssetBundle container entries Editor-only assets (ShaderSubGraph, .preset) can appear in an AssetBundle's m_Container with a null PPtr (m_FileID 0 and m_PathID 0) because they have no runtime object. The non-scene branch resolved that to a phantom (file, 0) object id and wrote an assetbundle_assets row pointing at it, which then surfaced as a false dangling reference with object_id 0 into an already-analyzed file. Skip container entries whose PPtr is null. assetbundle_asset_view already dropped these rows via its INNER JOIN, so this only removes rows that pointed at a non-existent object. Verified on a large Addressables build: the false object_id 0 dangling refs are gone, leaving only the expected unity default resources targets. --- Analyzer/SQLite/Handlers/AssetBundleHandler.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Analyzer/SQLite/Handlers/AssetBundleHandler.cs b/Analyzer/SQLite/Handlers/AssetBundleHandler.cs index 2d5552c..85e963d 100644 --- a/Analyzer/SQLite/Handlers/AssetBundleHandler.cs +++ b/Analyzer/SQLite/Handlers/AssetBundleHandler.cs @@ -68,6 +68,13 @@ public void Process(Context ctx, long objectId, RandomAccessReader reader, out s { if (!assetBundle.IsSceneAssetBundle) { + // Editor-only container entries (e.g. ShaderSubGraph, .preset) can appear in + // m_Container with a null PPtr (m_FileID 0 and m_PathID 0) because they have no + // runtime object. Skip them: resolving the null PPtr would allocate a phantom + // (file, 0) object id and record a bogus dangling reference to it. + if (asset.PPtr.FileId == 0 && asset.PPtr.PathId == 0) + continue; + var fileId = ctx.LocalToDbFileId[asset.PPtr.FileId]; var objId = ctx.ObjectIdProvider.GetId((fileId, asset.PPtr.PathId)); m_InsertCommand.Transaction = ctx.Transaction; From fc9d4fb3407be6c2e29a892d0315a2cebd53d895 Mon Sep 17 00:00:00 2001 From: Andrew Skowronski <86242170+SkowronskiAndrew@users.noreply.github.com> Date: Tue, 14 Jul 2026 12:55:22 -0400 Subject: [PATCH 4/4] Change wording of a comment Co-authored-by: Andrew Skowronski <86242170+SkowronskiAndrew@users.noreply.github.com> --- Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs b/Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs index b354c11..e5bb82d 100644 --- a/Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs +++ b/Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs @@ -367,7 +367,9 @@ public void FinalizeDatabase() var (fileId, pathId) = entry.Key; // Ensure the (un-analyzed) target file has a serialized_files row so the dangling - // ref can name it. archive is NULL: the file was referenced, not opened. + // ref can name it. We have to put null for archive - even if the target file is inside an AssetBundle + // or other archive file, because we simply don't have that information. This is fundamental to + // how references work in Unity. if (writtenFileIds.Add(fileId)) { m_AddSerializedFileCommand.SetTransaction(transaction);