[#85] Track dangling references in analyze

Analyzer/AnalyzerTool.cs

@@ -5,8 +5,8 @@
 using UnityDataTools.Analyzer.SQLite.Handlers;
 using UnityDataTools.Analyzer.SQLite.Parsers;
 using UnityDataTools.Analyzer.SQLite.Writers;
-using UnityDataTools.Models;
 using UnityDataTools.FileSystem;
+using UnityDataTools.Models;
 
 namespace UnityDataTools.Analyzer;
 
@@ -131,6 +131,13 @@ public int Analyze(AnalyzeOptions options)
         Console.WriteLine();
         Console.WriteLine($"Finalizing database. Successfully processed files: {countSuccess}, Failed files: {countFailures}, Files without TypeTrees: {countNoTypeTrees}, Ignored files: {countIgnored}");
 
+        // Record data that can only be determined once every file has been processed (e.g. which
+        // referenced objects were never resolved) before the database is finalized.
+        foreach (var parser in parsers)
+        {
+            parser.FinalizeDatabase();
+        }
+
         writer.End();
         foreach (var parser in parsers)
         {

Analyzer/Resources/Init.sql

@@ -81,13 +81,53 @@ CREATE TABLE IF NOT EXISTS refs
     property_type INTEGER
 );
 
+-- One row per referenced object that analyze assigned an id to but never wrote an objects row
+-- for (its serialized file was not part of the analyzed input). Written after all files are
+-- processed, once we can tell which assigned ids never became objects. Common causes: analyzing a
+-- partial set of AssetBundles, references into "unity default resources" (shipped without
+-- TypeTrees), or a ContentDirectory build analyzed without ContentLayout.json.
+-- Columns mirror the objects table so every object id resolves to exactly one of objects or
+-- dangling_refs:
+--   id - the analyzer object id (no row in objects has this id).
+--   object_id - the target's local file id (LFID / m_PathID) within its serialized file.
+--   serialized_file - references the serialized_files row of the (un-analyzed) file the target
+--        lives in. That row has archive NULL and no objects of its own.
+CREATE TABLE IF NOT EXISTS dangling_refs
+(
+    id INTEGER,
+    object_id INTEGER,
+    serialized_file INTEGER,
+    PRIMARY KEY (id)
+);
+
 -- Resolves the property_path and property_type ids in the refs table to their string values.
 CREATE VIEW refs_view AS
 SELECT r.object, r.referenced_object, pn.name AS property_path, pt.name AS property_type
 FROM refs r
 INNER JOIN property_names pn ON r.property_path = pn.id
 INNER JOIN property_types pt ON r.property_type = pt.id;
 
+-- Resolves dangling_refs to the source object(s) that reference each missing target, one row per
+-- (referencing object -> dangling target) reference. Not populated when analyze is run with
+-- --skip-references (neither refs nor dangling_refs are populated in that mode).
+CREATE VIEW dangling_refs_view AS
+SELECT
+    r.object AS source_id,
+    src_sf.name AS source_serialized_file,
+    src_o.object_id AS source_object_id,
+    pn.name AS property_path,
+    pt.name AS property_type,
+    d.id AS target_id,
+    tgt_sf.name AS target_serialized_file,
+    d.object_id AS target_object_id
+FROM dangling_refs d
+INNER JOIN refs r ON r.referenced_object = d.id
+INNER JOIN objects src_o ON r.object = src_o.id
+INNER JOIN serialized_files src_sf ON src_o.serialized_file = src_sf.id
+INNER JOIN serialized_files tgt_sf ON d.serialized_file = tgt_sf.id
+LEFT JOIN property_names pn ON r.property_path = pn.id
+LEFT JOIN property_types pt ON r.property_type = pt.id;
+
 CREATE VIEW object_view AS
 SELECT o.id, o.object_id, ab.name AS archive, sf.name AS serialized_file, t.name AS type, o.name, o.game_object, o.size,
 CASE
@@ -157,8 +197,9 @@ INSERT INTO types (id, name) VALUES (-1, 'Scene');
 -- 1 = normalized refs table (issue #44); 2 = renamed assets/asset_dependencies tables to
 -- assetbundle_assets/preload_dependencies (issue #82); 3 = renamed asset_bundles table to archives
 -- and the asset_bundle column/alias to archive (issue #68); 4 = build_report_packed_asset_contents_view
--- type column changed from numeric id to type name (issue #55); databases produced before versioning report 0.
-PRAGMA user_version = 4;
+-- type column changed from numeric id to type name (issue #55); 5 = added dangling_refs table/view
+-- (issue #85); databases produced before versioning report 0.
+PRAGMA user_version = 5;
 
 PRAGMA synchronous = OFF;
 PRAGMA journal_mode = MEMORY;

Analyzer/SQLite/Commands/SerializedFile/AddDanglingRef.cs

@@ -0,0 +1,29 @@
+using System.Collections.Generic;
+using Microsoft.Data.Sqlite;
+using UnityDataTools.Analyzer.SQLite.Commands;
+
+namespace UnityDataTools.Analyzer.SQLite.Commands.SerializedFile
+{
+    /* TABLE DEFINITION:
+    create table dangling_refs
+    (
+        id INTEGER,
+        object_id INTEGER,
+        serialized_file INTEGER,
+        PRIMARY KEY (id)
+    );
+    */
+    internal class AddDanglingRef : AbstractCommand
+    {
+        protected override string TableName => "dangling_refs";
+
+        protected override string DDLSource => null;
+
+        protected override Dictionary<string, SqliteType> Fields => new()
+        {
+            { "id", SqliteType.Integer },
+            { "object_id", SqliteType.Integer },
+            { "serialized_file", SqliteType.Integer }
+        };
+    }
+}

Analyzer/SQLite/Handlers/AssetBundleHandler.cs

@@ -68,6 +68,13 @@ public void Process(Context ctx, long objectId, RandomAccessReader reader, out s
         {
             if (!assetBundle.IsSceneAssetBundle)
             {
+                // Editor-only container entries (e.g. ShaderSubGraph, .preset) can appear in
+                // m_Container with a null PPtr (m_FileID 0 and m_PathID 0) because they have no
+                // runtime object. Skip them: resolving the null PPtr would allocate a phantom
+                // (file, 0) object id and record a bogus dangling reference to it.
+                if (asset.PPtr.FileId == 0 && asset.PPtr.PathId == 0)
+                    continue;
+
                 var fileId = ctx.LocalToDbFileId[asset.PPtr.FileId];
                 var objId = ctx.ObjectIdProvider.GetId((fileId, asset.PPtr.PathId));
                 m_InsertCommand.Transaction = ctx.Transaction;

Analyzer/SQLite/Handlers/ISQLiteHandler.cs

@@ -27,6 +27,12 @@ public interface ISQLiteFileParser : IDisposable
     void Init(SqliteConnection db);
     bool CanParse(string filename);
     void Parse(string filename);
+
+    // Called once after all files have been parsed, so a parser can write data that can only be
+    // determined from the complete set (e.g. dangling references). No-op for parsers that don't
+    // need it.
+    void FinalizeDatabase();
+
     public bool Verbose { get; set; }
     public bool SkipReferences { get; set; }
     public bool SkipCrc { get; set; }

Analyzer/SQLite/Parsers/AddressablesBuildLayoutParser.cs

@@ -4,8 +4,8 @@
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using UnityDataTools.Analyzer.SQLite.Handlers;
-using UnityDataTools.Models;
 using UnityDataTools.Analyzer.SQLite.Writers;
+using UnityDataTools.Models;
 
 namespace UnityDataTools.Analyzer.SQLite.Parsers
 {
@@ -21,6 +21,12 @@ public void Dispose()
         {
             m_Writer.Dispose();
         }
+
+        public void FinalizeDatabase()
+        {
+            // Addressables build reports don't produce dangling object references.
+        }
+
         public void Init(SqliteConnection db)
         {
             m_Writer = new AddressablesBuildLayoutSQLWriter(db);

Analyzer/SQLite/Parsers/SerializedFileParser.cs

@@ -35,6 +35,12 @@ public void Dispose()
             m_Writer.Dispose();
         }
 
+        public void FinalizeDatabase()
+        {
+            // m_Writer is only Init'd once a file is actually parsed; nothing to finalize otherwise.
+            m_Writer.FinalizeDatabase();
+        }
+
         public void Init(SqliteConnection db)
         {
             m_Writer = new SerializedFileSQLiteWriter(db, SkipReferences, SkipCrc);

Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs

@@ -74,6 +74,7 @@ public class SerializedFileSQLiteWriter : IDisposable
     private AddObject m_AddObjectCommand = new AddObject();
     private AddType m_AddTypeCommand = new AddType();
     private AddPreloadDependency m_InsertDepCommand = new AddPreloadDependency();
+    private AddDanglingRef m_AddDanglingRefCommand = new AddDanglingRef();
 
     private bool m_Initialized;
     private SqliteConnection m_Database;
@@ -112,6 +113,7 @@ private void CreateSQLiteCommands()
         m_AddObjectCommand.CreateCommand(m_Database);
         m_AddTypeCommand.CreateCommand(m_Database);
         m_InsertDepCommand.CreateCommand(m_Database);
+        m_AddDanglingRefCommand.CreateCommand(m_Database);
 
         m_LastId = m_Database.CreateCommand();
         m_LastId.CommandText = "SELECT last_insert_rowid()";
@@ -277,17 +279,22 @@ public void WriteSerializedFile(string relativePath, string fullPath, string con
                     name = randomAccessReader["m_Name"].GetValue<string>();
                 }
 
+                // Resolve m_GameObject to its analyzer object id for the game_object column, but only
+                // when the PPtr is non-null. A null PPtr (m_FileID 0 and m_PathID 0) is expected for
+                // any object that is not a component; resolving it would allocate a phantom id for a
+                // (file, 0) object that never exists and then surface as a bogus dangling ref.
+                object gameObject = "";
                 if (randomAccessReader.HasChild("m_GameObject"))
                 {
                     var pptr = randomAccessReader["m_GameObject"];
-                    var fileId = m_LocalToDbFileId[pptr["m_FileID"].GetValue<int>()];
-                    var gameObjectID = m_ObjectIdProvider.GetId((fileId, pptr["m_PathID"].GetValue<long>()));
-                    m_AddObjectCommand.SetValue("game_object", gameObjectID);
-                }
-                else
-                {
-                    m_AddObjectCommand.SetValue("game_object", "");
+                    var gameObjectFileId = pptr["m_FileID"].GetValue<int>();
+                    var gameObjectPathId = pptr["m_PathID"].GetValue<long>();
+                    if (gameObjectFileId != 0 || gameObjectPathId != 0)
+                    {
+                        gameObject = m_ObjectIdProvider.GetId((m_LocalToDbFileId[gameObjectFileId], gameObjectPathId));
+                    }
                 }
+                m_AddObjectCommand.SetValue("game_object", gameObject);
 
                 // The walk both extracts references and accumulates the CRC, so it is needed
                 // unless both are disabled. When CRC is on but references are off, the walk
@@ -328,6 +335,78 @@ public void WriteSerializedFile(string relativePath, string fullPath, string con
         }
     }
 
+    // Records every referenced object that was assigned an id but never written to the objects
+    // table (its serialized file was not part of the analyzed input). Must run after all files are
+    // processed, since an id is only known to be dangling once nothing has claimed it as an object.
+    // The set of written objects/files is read back from the database so this is robust to every
+    // object write site (regular objects, synthetic Scene objects, etc.) without instrumenting them.
+    public void FinalizeDatabase()
+    {
+        // Dangling refs are part of reference tracking, so honor --skip-references (the view that
+        // makes them useful joins the refs table, which is empty in that mode anyway).
+        if (!m_Initialized || m_SkipReferences)
+            return;
+
+        var writtenObjectIds = ReadIntSet("SELECT id FROM objects");
+        var writtenFileIds = ReadIntSet("SELECT id FROM serialized_files");
+
+        // Invert the file-name provider so a dangling target's file id maps back to its name.
+        var fileIdToName = new Dictionary<int, string>();
+        foreach (var entry in m_SerializedFileIdProvider.Entries)
+            fileIdToName[entry.Value] = entry.Key;
+
+        using var transaction = m_Database.BeginTransaction();
+        try
+        {
+            foreach (var entry in m_ObjectIdProvider.Entries)
+            {
+                var objectId = entry.Value;
+                if (writtenObjectIds.Contains(objectId))
+                    continue;
+
+                var (fileId, pathId) = entry.Key;
+
+                // Ensure the (un-analyzed) target file has a serialized_files row so the dangling
+                // ref can name it. We have to put null for archive - even if the target file is inside an AssetBundle
+                // or other archive file, because we simply don't have that information.  This is fundamental to
+                // how references work in Unity.
+                if (writtenFileIds.Add(fileId))
+                {
+                    m_AddSerializedFileCommand.SetTransaction(transaction);
+                    m_AddSerializedFileCommand.SetValue("id", fileId);
+                    m_AddSerializedFileCommand.SetValue("archive", null);
+                    m_AddSerializedFileCommand.SetValue("name",
+                        fileIdToName.TryGetValue(fileId, out var name) ? name : "");
+                    m_AddSerializedFileCommand.ExecuteNonQuery();
+                }
+
+                m_AddDanglingRefCommand.SetTransaction(transaction);
+                m_AddDanglingRefCommand.SetValue("id", objectId);
+                m_AddDanglingRefCommand.SetValue("object_id", pathId);
+                m_AddDanglingRefCommand.SetValue("serialized_file", fileId);
+                m_AddDanglingRefCommand.ExecuteNonQuery();
+            }
+
+            transaction.Commit();
+        }
+        catch (Exception)
+        {
+            transaction.Rollback();
+            throw;
+        }
+    }
+
+    private HashSet<int> ReadIntSet(string sql)
+    {
+        var result = new HashSet<int>();
+        using var command = m_Database.CreateCommand();
+        command.CommandText = sql;
+        using var reader = command.ExecuteReader();
+        while (reader.Read())
+            result.Add(reader.GetInt32(0));
+        return result;
+    }
+
     // Callback from PPtrAndCrcProcessor for each reference discovered in the SerializedFile
     private int AddReference(long objectId, int fileId, long pathId, string propertyPath, string propertyType)
     {
@@ -395,6 +474,7 @@ public void Dispose()
         m_AddObjectCommand.Dispose();
         m_AddTypeCommand.Dispose();
         m_InsertDepCommand.Dispose();
+        m_AddDanglingRefCommand.Dispose();
 
         m_LastId.Dispose();
     }

Analyzer/Util/IdProvider.cs

@@ -11,6 +11,10 @@ public class IdProvider<Key>
 {
     private Dictionary<Key, int> m_Ids = new();
 
+    // Exposes the key->id assignments so callers can iterate or invert the mapping (used at
+    // finalize to map a dangling object id back to its (fileId, pathId) or file name).
+    public IReadOnlyDictionary<Key, int> Entries => m_Ids;
+
     public int GetId(Key key)
     {
         if (m_Ids.TryGetValue(key, out var id))

Documentation/analyzer.md

@@ -98,7 +98,8 @@ What the `object` is depends on the build:
   `sharedassetsN.assets`) plus one in `globalgamemanagers.assets` for the always-loaded set.
 
 The `dependency` side can reference an object that analyze never recorded, leaving a "dangling" id
-with no matching `objects` row. The most common case is objects in `unity default resources`, the
+with no matching `objects` row (such ids are catalogued in the `dangling_refs` table). The most
+common case is objects in `unity default resources`, the
 built-in resource file that ships with the Unity Editor without TypeTrees and so cannot be analyzed
 without a specially built copy. (`Resources/unity_builtin_extra` is built alongside your content and
 can be analyzed, but produces the same dangling references when it is not part of the analyzed set.)
@@ -251,6 +252,36 @@ SELECT * FROM refs_view WHERE property_type = 'MonoScript';
 
 These tables are not populated when analyze is run with `--skip-references`.
 
+## dangling_refs / dangling_refs_view
+
+When a reference points to an object whose serialized file was not part of the analyzed input,
+analyze still assigns that target an object id but never writes an `objects` row for it. `dangling_refs`
+records those targets so the reference information is preserved and every object id is accounted for
+(each id resolves to exactly one of `objects` or `dangling_refs`, never both). Common causes:
+
+* Analyzing a single AssetBundle or a partial subset of a bundle group, so cross-bundle references
+  point at bundles that were not analyzed.
+* A Player build referencing `unity default resources` (and `Resources/unity_builtin_extra` when it
+  is not part of the analyzed set) - built-in files that ship without TypeTrees and are not analyzed.
+* A ContentDirectory build analyzed without its `ContentLayout.json`, so references cannot be resolved.
+
+The columns mirror the `objects` table: `id` (the assigned object id, absent from `objects`),
+`object_id` (the target's local file id / LFID) and `serialized_file`. The target file is recorded in
+`serialized_files` (with `archive` NULL and no objects of its own) so its name is available; the
+lowercased file name is all that is known about it - there is no type, size, or other detail.
+
+`dangling_refs_view` joins each dangling target back to the object(s) that reference it, one row per
+reference, with columns `source_id`, `source_serialized_file`, `source_object_id`, `property_path`,
+`property_type`, `target_id`, `target_serialized_file`, `target_object_id`.
+
+```sql
+-- what does object 42 fail to resolve, and where should those objects have come from?
+SELECT * FROM dangling_refs_view WHERE source_id = 42;
+```
+
+Because the view joins `refs`, it is not populated when analyze is run with `--skip-references`
+(in that mode neither `refs` nor `dangling_refs` are populated).
+
 ## BuildReport
 
 See [BuildReport.md](buildreport.md) for details of the tables and views related to analyzing BuildReport files.