[#99] Import ContentLayout.json into the analyze database

Analyzer/AnalyzerTool.cs

@@ -16,6 +16,7 @@ public class AnalyzerTool
 
     public List<ISQLiteFileParser> parsers = new List<ISQLiteFileParser>()
     {
+        new ContentLayoutParser(),
         new AddressablesBuildLayoutParser(),
         new SerializedFileParser(),
     };

Analyzer/Properties/Resources.resx

@@ -226,6 +226,42 @@
   <data name="AddrBuildSchemaDataPairs" type="System.Resources.ResXFileRef, System.Windows.Forms">
     <value>..\Resources\AddrBuildSchemaDataPairs.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8</value>
   </data>
+  <data name="ContentLayout" type="System.Resources.ResXFileRef, System.Windows.Forms">
+    <value>..\Resources\ContentLayout.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8</value>
+  </data>
+  <data name="ContentLayoutSerializedFiles" type="System.Resources.ResXFileRef, System.Windows.Forms">
+    <value>..\Resources\ContentLayoutSerializedFiles.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8</value>
+  </data>
+  <data name="ContentLayoutSourceAssets" type="System.Resources.ResXFileRef, System.Windows.Forms">
+    <value>..\Resources\ContentLayoutSourceAssets.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8</value>
+  </data>
+  <data name="ContentLayoutSerializedFileDependencies" type="System.Resources.ResXFileRef, System.Windows.Forms">
+    <value>..\Resources\ContentLayoutSerializedFileDependencies.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8</value>
+  </data>
+  <data name="ContentLayoutLoadableDependencies" type="System.Resources.ResXFileRef, System.Windows.Forms">
+    <value>..\Resources\ContentLayoutLoadableDependencies.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8</value>
+  </data>
+  <data name="ContentLayoutLoadableSceneDependencies" type="System.Resources.ResXFileRef, System.Windows.Forms">
+    <value>..\Resources\ContentLayoutLoadableSceneDependencies.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8</value>
+  </data>
+  <data name="ContentLayoutLoadableObjects" type="System.Resources.ResXFileRef, System.Windows.Forms">
+    <value>..\Resources\ContentLayoutLoadableObjects.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8</value>
+  </data>
+  <data name="ContentLayoutLoadableScenes" type="System.Resources.ResXFileRef, System.Windows.Forms">
+    <value>..\Resources\ContentLayoutLoadableScenes.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8</value>
+  </data>
+  <data name="ContentLayoutBinaryArtifacts" type="System.Resources.ResXFileRef, System.Windows.Forms">
+    <value>..\Resources\ContentLayoutBinaryArtifacts.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8</value>
+  </data>
+  <data name="ContentLayoutArtifactReferences" type="System.Resources.ResXFileRef, System.Windows.Forms">
+    <value>..\Resources\ContentLayoutArtifactReferences.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8</value>
+  </data>
+  <data name="ContentLayoutViews" type="System.Resources.ResXFileRef, System.Windows.Forms">
+    <value>..\Resources\ContentLayoutViews.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8</value>
+  </data>
+  <data name="ContentLayoutIndexes" type="System.Resources.ResXFileRef, System.Windows.Forms">
+    <value>..\Resources\ContentLayoutIndexes.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8</value>
+  </data>
   <data name="MonoScript" type="System.Resources.ResXFileRef, System.Windows.Forms">
     <value>..\Resources\MonoScript.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8</value>
   </data>

Analyzer/Resources/ContentLayout.sql

@@ -0,0 +1,11 @@
+-- Identity of the imported ContentLayout.json (see Documentation/contentlayout.md). The
+-- content_layout* tables are only created when a ContentLayout.json is part of the analyzed
+-- input. A single layout per database is supported.
+CREATE TABLE IF NOT EXISTS content_layout
+(
+    id INTEGER,                 -- always 0 (single layout per database)
+    name TEXT,                  -- path of the imported ContentLayout.json
+    version INTEGER,            -- schema version of the json file
+    build_manifest_hash TEXT,
+    PRIMARY KEY (id)
+);

Analyzer/Resources/ContentLayoutArtifactReferences.sql

@@ -0,0 +1,10 @@
+-- Direct references between binary artifacts (ArtifactReferences in the json), e.g. a
+-- serialized file referencing its .resS/.resource data files. References that go through a
+-- loadable are not included, and the graph is never cyclical. References to other serialized
+-- files are not recorded here either; those are in content_layout_serialized_file_dependencies.
+CREATE TABLE IF NOT EXISTS content_layout_artifact_references
+(
+    artifact_index INTEGER,           -- references content_layout_binary_artifacts.artifact_index
+    referenced_artifact_index INTEGER,
+    PRIMARY KEY (artifact_index, referenced_artifact_index)
+);

Analyzer/Resources/ContentLayoutBinaryArtifacts.sql

@@ -0,0 +1,12 @@
+-- The artifacts that make up the build output (BinaryArtifacts in the json): the serialized
+-- files themselves plus the data files they use (.resS, .resource) and the manifest. This is the
+-- standard place to find artifact sizes. When stored as a file the filename is the content hash
+-- plus an extension derived from the category (content_layout_binary_artifacts_view adds it).
+CREATE TABLE IF NOT EXISTS content_layout_binary_artifacts
+(
+    artifact_index INTEGER,
+    content_hash TEXT,
+    category TEXT,              -- 'texture' | 'mesh' | 'audio' | 'video' | 'contentfile' | 'manifest'
+    size INTEGER,
+    PRIMARY KEY (artifact_index)
+);

Analyzer/Resources/ContentLayoutIndexes.sql

@@ -0,0 +1,10 @@
+-- Created after the content_layout tables are populated, so inserts stay fast for very large
+-- layouts. The content_hash and asset_path indexes carry the views; the rest serve reverse
+-- lookups ("who depends on X", "which loadables live in file Y").
+CREATE INDEX content_layout_binary_artifacts_content_hash ON content_layout_binary_artifacts(content_hash);
+CREATE INDEX content_layout_source_assets_asset_path ON content_layout_source_assets(asset_path);
+CREATE INDEX content_layout_source_assets_file ON content_layout_source_assets(serialized_file_index);
+CREATE INDEX content_layout_serialized_file_dependencies_dep ON content_layout_serialized_file_dependencies(dependency_index);
+CREATE INDEX content_layout_artifact_references_ref ON content_layout_artifact_references(referenced_artifact_index);
+CREATE INDEX content_layout_loadable_objects_file ON content_layout_loadable_objects(serialized_file_index);
+CREATE INDEX content_layout_loadable_objects_guid ON content_layout_loadable_objects(guid);

Analyzer/Resources/ContentLayoutLoadableDependencies.sql

@@ -0,0 +1,6 @@
+-- Loadable objects referenced from each serialized file (LoadableDependencies in the json).
+CREATE TABLE IF NOT EXISTS content_layout_loadable_dependencies
+(
+    serialized_file_index INTEGER,   -- references content_layout_serialized_files.file_index
+    object_id_hash TEXT              -- references content_layout_loadable_objects.object_id_hash
+);

Analyzer/Resources/ContentLayoutLoadableObjects.sql

@@ -0,0 +1,17 @@
+-- The objects that can be loaded on demand (LoadableObjectIds in the json), identified
+-- independently of the serialized file that contains them. Also records where each one came from
+-- in the source project. The json's top-level RootAssets list is folded into the is_root_asset
+-- flag. serialized_file_index is NULL when the object was dropped from the build (json value -1,
+-- e.g. server build shader references).
+CREATE TABLE IF NOT EXISTS content_layout_loadable_objects
+(
+    object_id_hash TEXT,             -- hash of GUID, LFID and identifier_type
+    guid TEXT,                       -- AssetDatabase GUID of the source asset
+    asset_path TEXT,
+    lfid INTEGER,                    -- local file id of the object in the source asset
+    identifier_type INTEGER,
+    serialized_file_index INTEGER,   -- references content_layout_serialized_files.file_index, or NULL
+    output_lfid INTEGER,             -- local file id of the object in its output serialized file
+    is_root_asset INTEGER,
+    PRIMARY KEY (object_id_hash)
+);

Analyzer/Resources/ContentLayoutLoadableSceneDependencies.sql

@@ -0,0 +1,6 @@
+-- Scenes referenced from each serialized file (LoadableSceneDependencies in the json).
+CREATE TABLE IF NOT EXISTS content_layout_loadable_scene_dependencies
+(
+    serialized_file_index INTEGER,   -- references content_layout_serialized_files.file_index
+    scene_path TEXT                  -- matches content_layout_loadable_scenes.path
+);

Analyzer/Resources/ContentLayoutLoadableScenes.sql

@@ -0,0 +1,8 @@
+-- The scenes exposed as loadable in the build (LoadableSceneIds in the json).
+CREATE TABLE IF NOT EXISTS content_layout_loadable_scenes
+(
+    guid TEXT,
+    path TEXT,
+    serialized_file_index INTEGER,   -- references content_layout_serialized_files.file_index, or NULL
+    PRIMARY KEY (guid)
+);

Analyzer/Resources/ContentLayoutSerializedFileDependencies.sql

@@ -0,0 +1,11 @@
+-- File-to-file dependency edges (SerializedFileDependencies in the json): the other serialized
+-- files that must be loaded before this one. position preserves the array order, which is
+-- significant: a PPtr's m_FileID inside the file resolves positionally through this list (see
+-- Documentation/contentdirectory-format.md).
+CREATE TABLE IF NOT EXISTS content_layout_serialized_file_dependencies
+(
+    serialized_file_index INTEGER,   -- references content_layout_serialized_files.file_index
+    position INTEGER,                -- 1-based, matching the external reference table / m_FileID index
+    dependency_index INTEGER,        -- references content_layout_serialized_files.file_index
+    PRIMARY KEY (serialized_file_index, position)
+);

Analyzer/Resources/ContentLayoutSerializedFiles.sql

@@ -0,0 +1,15 @@
+-- One row per entry in the layout's SerializedFiles array: the serialized files (.cf Content
+-- Files) that the ContentDirectory build produced. file_index is the array index from the json
+-- and is how the other content_layout tables reference a file.
+-- serialized_file links to the core serialized_files table so layout data joins directly with
+-- analyzed objects and references; it is NULL when the analyzed input did not include the build
+-- content (e.g. a layout-only analyze) or for built-in entries.
+CREATE TABLE IF NOT EXISTS content_layout_serialized_files
+(
+    file_index INTEGER,
+    cfid TEXT,                  -- symbolic .cfid reference string (for built-ins: the built-in path)
+    is_builtin INTEGER,
+    content_hash TEXT,          -- NULL for built-ins; the filename is content_hash || '.cf'
+    serialized_file INTEGER,    -- references serialized_files.id, or NULL
+    PRIMARY KEY (file_index)
+);

Analyzer/Resources/ContentLayoutSourceAssets.sql

@@ -0,0 +1,7 @@
+-- The source assets included in each serialized file (SourceAssets in the json). The same asset
+-- path can appear in more than one file (e.g. an FBX split into multiple output files).
+CREATE TABLE IF NOT EXISTS content_layout_source_assets
+(
+    serialized_file_index INTEGER,   -- references content_layout_serialized_files.file_index
+    asset_path TEXT
+);

Analyzer/Resources/ContentLayoutViews.sql

@@ -0,0 +1,64 @@
+-- Convenience views over the content_layout tables. Created together with the tables (only when
+-- a ContentLayout.json is imported). See Documentation/contentlayout-database.md for the schema
+-- reference.
+
+-- One row per layout serialized file with the derived filename, artifact size, and core-table link.
+-- Built-in entries have no file on disk, so their path (cfid) is shown as the filename.
+CREATE VIEW IF NOT EXISTS content_layout_serialized_files_view AS
+SELECT f.file_index, f.cfid, f.is_builtin,
+       CASE WHEN f.is_builtin = 1 THEN f.cfid ELSE f.content_hash || '.cf' END AS filename,
+       ba.size,
+       f.serialized_file,
+       sf.archive
+FROM content_layout_serialized_files f
+LEFT JOIN content_layout_binary_artifacts ba ON ba.content_hash = f.content_hash AND ba.category = 'contentfile'
+LEFT JOIN serialized_files sf ON sf.id = f.serialized_file;
+
+-- Source asset -> the file(s) it was built into. Built-in entries have no source assets, so no
+-- filename fallback is needed here.
+CREATE VIEW IF NOT EXISTS content_layout_source_assets_view AS
+SELECT s.asset_path, f.file_index, f.content_hash || '.cf' AS filename, f.serialized_file
+FROM content_layout_source_assets s
+INNER JOIN content_layout_serialized_files f ON f.file_index = s.serialized_file_index;
+
+-- File-to-file dependency edges with names resolved on both sides (via the files view, so
+-- dependencies on built-in entries show their path instead of a NULL filename).
+CREATE VIEW IF NOT EXISTS content_layout_serialized_file_dependencies_view AS
+SELECT d.serialized_file_index, src.filename, d.position,
+       d.dependency_index, dep.filename AS dependency_filename, dep.cfid AS dependency_cfid
+FROM content_layout_serialized_file_dependencies d
+INNER JOIN content_layout_serialized_files_view src ON src.file_index = d.serialized_file_index
+INNER JOIN content_layout_serialized_files_view dep ON dep.file_index = d.dependency_index;
+
+-- Loadables resolved to their analyzed object (object columns are NULL in a layout-only database).
+CREATE VIEW IF NOT EXISTS content_layout_loadable_objects_view AS
+SELECT l.object_id_hash, l.guid, l.asset_path, l.lfid, l.is_root_asset,
+       f.content_hash || '.cf' AS filename,
+       o.id AS object, t.name AS type, o.name, o.size
+FROM content_layout_loadable_objects l
+LEFT JOIN content_layout_serialized_files f ON f.file_index = l.serialized_file_index
+LEFT JOIN objects o ON o.serialized_file = f.serialized_file AND o.object_id = l.output_lfid
+LEFT JOIN types t ON t.id = o.type;
+
+-- Artifacts with their derived on-disk filename (extension based on the category).
+CREATE VIEW IF NOT EXISTS content_layout_binary_artifacts_view AS
+SELECT artifact_index, content_hash, category, size,
+       content_hash ||
+       CASE category
+           WHEN 'texture' THEN '.resS'   WHEN 'mesh' THEN '.resS'
+           WHEN 'audio' THEN '.resource' WHEN 'video' THEN '.resource'
+           WHEN 'contentfile' THEN '.cf' WHEN 'manifest' THEN '.json'
+           ELSE '.' || category
+       END AS filename
+FROM content_layout_binary_artifacts;
+
+-- The data files (.resS/.resource) each serialized file uses, derived from the artifact graph.
+CREATE VIEW IF NOT EXISTS content_layout_data_files_view AS
+SELECT f.file_index, f.content_hash || '.cf' AS filename,
+       ra.category, rav.filename AS data_filename, ra.size
+FROM content_layout_serialized_files f
+INNER JOIN content_layout_binary_artifacts ca ON ca.content_hash = f.content_hash AND ca.category = 'contentfile'
+INNER JOIN content_layout_artifact_references r ON r.artifact_index = ca.artifact_index
+INNER JOIN content_layout_binary_artifacts ra ON ra.artifact_index = r.referenced_artifact_index
+INNER JOIN content_layout_binary_artifacts_view rav ON rav.artifact_index = ra.artifact_index
+WHERE ra.category IN ('texture', 'mesh', 'audio', 'video');

Analyzer/SQLite/Commands/ContentLayout/AddContentLayout.cs

@@ -0,0 +1,20 @@
+using System.Collections.Generic;
+using Microsoft.Data.Sqlite;
+
+namespace UnityDataTools.Analyzer.SQLite.Commands.ContentLayout
+{
+    internal class AddContentLayout : AbstractCommand
+    {
+        protected override string TableName => "content_layout";
+
+        protected override string DDLSource => Properties.Resources.ContentLayout;
+
+        protected override Dictionary<string, SqliteType> Fields => new()
+        {
+            { "id", SqliteType.Integer },
+            { "name", SqliteType.Text },
+            { "version", SqliteType.Integer },
+            { "build_manifest_hash", SqliteType.Text }
+        };
+    }
+}

Analyzer/SQLite/Commands/ContentLayout/AddContentLayoutArtifactReference.cs

@@ -0,0 +1,18 @@
+using System.Collections.Generic;
+using Microsoft.Data.Sqlite;
+
+namespace UnityDataTools.Analyzer.SQLite.Commands.ContentLayout
+{
+    internal class AddContentLayoutArtifactReference : AbstractCommand
+    {
+        protected override string TableName => "content_layout_artifact_references";
+
+        protected override string DDLSource => Properties.Resources.ContentLayoutArtifactReferences;
+
+        protected override Dictionary<string, SqliteType> Fields => new()
+        {
+            { "artifact_index", SqliteType.Integer },
+            { "referenced_artifact_index", SqliteType.Integer }
+        };
+    }
+}

Analyzer/SQLite/Commands/ContentLayout/AddContentLayoutBinaryArtifact.cs

@@ -0,0 +1,20 @@
+using System.Collections.Generic;
+using Microsoft.Data.Sqlite;
+
+namespace UnityDataTools.Analyzer.SQLite.Commands.ContentLayout
+{
+    internal class AddContentLayoutBinaryArtifact : AbstractCommand
+    {
+        protected override string TableName => "content_layout_binary_artifacts";
+
+        protected override string DDLSource => Properties.Resources.ContentLayoutBinaryArtifacts;
+
+        protected override Dictionary<string, SqliteType> Fields => new()
+        {
+            { "artifact_index", SqliteType.Integer },
+            { "content_hash", SqliteType.Text },
+            { "category", SqliteType.Text },
+            { "size", SqliteType.Integer }
+        };
+    }
+}