diff --git a/Analyzer/AnalyzerTool.cs b/Analyzer/AnalyzerTool.cs index da8473c..5790f66 100644 --- a/Analyzer/AnalyzerTool.cs +++ b/Analyzer/AnalyzerTool.cs @@ -16,6 +16,7 @@ public class AnalyzerTool public List parsers = new List() { + new ContentLayoutParser(), new AddressablesBuildLayoutParser(), new SerializedFileParser(), }; diff --git a/Analyzer/Properties/Resources.Designer.cs b/Analyzer/Properties/Resources.Designer.cs index 57da7d6..57f5822 100644 --- a/Analyzer/Properties/Resources.Designer.cs +++ b/Analyzer/Properties/Resources.Designer.cs @@ -246,6 +246,78 @@ internal static string BuildReport { } } + internal static string ContentLayout { + get { + return ResourceManager.GetString("ContentLayout", resourceCulture); + } + } + + internal static string ContentLayoutArtifactReferences { + get { + return ResourceManager.GetString("ContentLayoutArtifactReferences", resourceCulture); + } + } + + internal static string ContentLayoutBinaryArtifacts { + get { + return ResourceManager.GetString("ContentLayoutBinaryArtifacts", resourceCulture); + } + } + + internal static string ContentLayoutIndexes { + get { + return ResourceManager.GetString("ContentLayoutIndexes", resourceCulture); + } + } + + internal static string ContentLayoutLoadableDependencies { + get { + return ResourceManager.GetString("ContentLayoutLoadableDependencies", resourceCulture); + } + } + + internal static string ContentLayoutLoadableObjects { + get { + return ResourceManager.GetString("ContentLayoutLoadableObjects", resourceCulture); + } + } + + internal static string ContentLayoutLoadableSceneDependencies { + get { + return ResourceManager.GetString("ContentLayoutLoadableSceneDependencies", resourceCulture); + } + } + + internal static string ContentLayoutLoadableScenes { + get { + return ResourceManager.GetString("ContentLayoutLoadableScenes", resourceCulture); + } + } + + internal static string ContentLayoutSerializedFileDependencies { + get { + return ResourceManager.GetString("ContentLayoutSerializedFileDependencies", resourceCulture); + } + } + + internal static string ContentLayoutSerializedFiles { + get { + return ResourceManager.GetString("ContentLayoutSerializedFiles", resourceCulture); + } + } + + internal static string ContentLayoutSourceAssets { + get { + return ResourceManager.GetString("ContentLayoutSourceAssets", resourceCulture); + } + } + + internal static string ContentLayoutViews { + get { + return ResourceManager.GetString("ContentLayoutViews", resourceCulture); + } + } + internal static string Finalize { get { return ResourceManager.GetString("Finalize", resourceCulture); diff --git a/Analyzer/Properties/Resources.resx b/Analyzer/Properties/Resources.resx index 1722f57..24d30da 100644 --- a/Analyzer/Properties/Resources.resx +++ b/Analyzer/Properties/Resources.resx @@ -226,6 +226,42 @@ ..\Resources\AddrBuildSchemaDataPairs.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8 + + ..\Resources\ContentLayout.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8 + + + ..\Resources\ContentLayoutSerializedFiles.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8 + + + ..\Resources\ContentLayoutSourceAssets.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8 + + + ..\Resources\ContentLayoutSerializedFileDependencies.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8 + + + ..\Resources\ContentLayoutLoadableDependencies.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8 + + + ..\Resources\ContentLayoutLoadableSceneDependencies.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8 + + + ..\Resources\ContentLayoutLoadableObjects.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8 + + + ..\Resources\ContentLayoutLoadableScenes.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8 + + + ..\Resources\ContentLayoutBinaryArtifacts.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8 + + + ..\Resources\ContentLayoutArtifactReferences.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8 + + + ..\Resources\ContentLayoutViews.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8 + + + ..\Resources\ContentLayoutIndexes.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8 + ..\Resources\MonoScript.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8 diff --git a/Analyzer/Resources/ContentLayout.sql b/Analyzer/Resources/ContentLayout.sql new file mode 100644 index 0000000..1896e4a --- /dev/null +++ b/Analyzer/Resources/ContentLayout.sql @@ -0,0 +1,11 @@ +-- Identity of the imported ContentLayout.json (see Documentation/contentlayout.md). The +-- content_layout* tables are only created when a ContentLayout.json is part of the analyzed +-- input. A single layout per database is supported. +CREATE TABLE IF NOT EXISTS content_layout +( + id INTEGER, -- always 0 (single layout per database) + name TEXT, -- path of the imported ContentLayout.json + version INTEGER, -- schema version of the json file + build_manifest_hash TEXT, + PRIMARY KEY (id) +); diff --git a/Analyzer/Resources/ContentLayoutArtifactReferences.sql b/Analyzer/Resources/ContentLayoutArtifactReferences.sql new file mode 100644 index 0000000..cf63fdd --- /dev/null +++ b/Analyzer/Resources/ContentLayoutArtifactReferences.sql @@ -0,0 +1,10 @@ +-- Direct references between binary artifacts (ArtifactReferences in the json), e.g. a +-- serialized file referencing its .resS/.resource data files. References that go through a +-- loadable are not included, and the graph is never cyclical. References to other serialized +-- files are not recorded here either; those are in content_layout_serialized_file_dependencies. +CREATE TABLE IF NOT EXISTS content_layout_artifact_references +( + artifact_index INTEGER, -- references content_layout_binary_artifacts.artifact_index + referenced_artifact_index INTEGER, + PRIMARY KEY (artifact_index, referenced_artifact_index) +); diff --git a/Analyzer/Resources/ContentLayoutBinaryArtifacts.sql b/Analyzer/Resources/ContentLayoutBinaryArtifacts.sql new file mode 100644 index 0000000..9fa21f0 --- /dev/null +++ b/Analyzer/Resources/ContentLayoutBinaryArtifacts.sql @@ -0,0 +1,12 @@ +-- The artifacts that make up the build output (BinaryArtifacts in the json): the serialized +-- files themselves plus the data files they use (.resS, .resource) and the manifest. This is the +-- standard place to find artifact sizes. When stored as a file the filename is the content hash +-- plus an extension derived from the category (content_layout_binary_artifacts_view adds it). +CREATE TABLE IF NOT EXISTS content_layout_binary_artifacts +( + artifact_index INTEGER, + content_hash TEXT, + category TEXT, -- 'texture' | 'mesh' | 'audio' | 'video' | 'contentfile' | 'manifest' + size INTEGER, + PRIMARY KEY (artifact_index) +); diff --git a/Analyzer/Resources/ContentLayoutIndexes.sql b/Analyzer/Resources/ContentLayoutIndexes.sql new file mode 100644 index 0000000..78dbf94 --- /dev/null +++ b/Analyzer/Resources/ContentLayoutIndexes.sql @@ -0,0 +1,10 @@ +-- Created after the content_layout tables are populated, so inserts stay fast for very large +-- layouts. The content_hash and asset_path indexes carry the views; the rest serve reverse +-- lookups ("who depends on X", "which loadables live in file Y"). +CREATE INDEX content_layout_binary_artifacts_content_hash ON content_layout_binary_artifacts(content_hash); +CREATE INDEX content_layout_source_assets_asset_path ON content_layout_source_assets(asset_path); +CREATE INDEX content_layout_source_assets_file ON content_layout_source_assets(serialized_file_index); +CREATE INDEX content_layout_serialized_file_dependencies_dep ON content_layout_serialized_file_dependencies(dependency_index); +CREATE INDEX content_layout_artifact_references_ref ON content_layout_artifact_references(referenced_artifact_index); +CREATE INDEX content_layout_loadable_objects_file ON content_layout_loadable_objects(serialized_file_index); +CREATE INDEX content_layout_loadable_objects_guid ON content_layout_loadable_objects(guid); diff --git a/Analyzer/Resources/ContentLayoutLoadableDependencies.sql b/Analyzer/Resources/ContentLayoutLoadableDependencies.sql new file mode 100644 index 0000000..4293a4f --- /dev/null +++ b/Analyzer/Resources/ContentLayoutLoadableDependencies.sql @@ -0,0 +1,6 @@ +-- Loadable objects referenced from each serialized file (LoadableDependencies in the json). +CREATE TABLE IF NOT EXISTS content_layout_loadable_dependencies +( + serialized_file_index INTEGER, -- references content_layout_serialized_files.file_index + object_id_hash TEXT -- references content_layout_loadable_objects.object_id_hash +); diff --git a/Analyzer/Resources/ContentLayoutLoadableObjects.sql b/Analyzer/Resources/ContentLayoutLoadableObjects.sql new file mode 100644 index 0000000..179465b --- /dev/null +++ b/Analyzer/Resources/ContentLayoutLoadableObjects.sql @@ -0,0 +1,17 @@ +-- The objects that can be loaded on demand (LoadableObjectIds in the json), identified +-- independently of the serialized file that contains them. Also records where each one came from +-- in the source project. The json's top-level RootAssets list is folded into the is_root_asset +-- flag. serialized_file_index is NULL when the object was dropped from the build (json value -1, +-- e.g. server build shader references). +CREATE TABLE IF NOT EXISTS content_layout_loadable_objects +( + object_id_hash TEXT, -- hash of GUID, LFID and identifier_type + guid TEXT, -- AssetDatabase GUID of the source asset + asset_path TEXT, + lfid INTEGER, -- local file id of the object in the source asset + identifier_type INTEGER, + serialized_file_index INTEGER, -- references content_layout_serialized_files.file_index, or NULL + output_lfid INTEGER, -- local file id of the object in its output serialized file + is_root_asset INTEGER, + PRIMARY KEY (object_id_hash) +); diff --git a/Analyzer/Resources/ContentLayoutLoadableSceneDependencies.sql b/Analyzer/Resources/ContentLayoutLoadableSceneDependencies.sql new file mode 100644 index 0000000..84c2eac --- /dev/null +++ b/Analyzer/Resources/ContentLayoutLoadableSceneDependencies.sql @@ -0,0 +1,6 @@ +-- Scenes referenced from each serialized file (LoadableSceneDependencies in the json). +CREATE TABLE IF NOT EXISTS content_layout_loadable_scene_dependencies +( + serialized_file_index INTEGER, -- references content_layout_serialized_files.file_index + scene_path TEXT -- matches content_layout_loadable_scenes.path +); diff --git a/Analyzer/Resources/ContentLayoutLoadableScenes.sql b/Analyzer/Resources/ContentLayoutLoadableScenes.sql new file mode 100644 index 0000000..7148b88 --- /dev/null +++ b/Analyzer/Resources/ContentLayoutLoadableScenes.sql @@ -0,0 +1,8 @@ +-- The scenes exposed as loadable in the build (LoadableSceneIds in the json). +CREATE TABLE IF NOT EXISTS content_layout_loadable_scenes +( + guid TEXT, + path TEXT, + serialized_file_index INTEGER, -- references content_layout_serialized_files.file_index, or NULL + PRIMARY KEY (guid) +); diff --git a/Analyzer/Resources/ContentLayoutSerializedFileDependencies.sql b/Analyzer/Resources/ContentLayoutSerializedFileDependencies.sql new file mode 100644 index 0000000..d71bdb1 --- /dev/null +++ b/Analyzer/Resources/ContentLayoutSerializedFileDependencies.sql @@ -0,0 +1,11 @@ +-- File-to-file dependency edges (SerializedFileDependencies in the json): the other serialized +-- files that must be loaded before this one. position preserves the array order, which is +-- significant: a PPtr's m_FileID inside the file resolves positionally through this list (see +-- Documentation/contentdirectory-format.md). +CREATE TABLE IF NOT EXISTS content_layout_serialized_file_dependencies +( + serialized_file_index INTEGER, -- references content_layout_serialized_files.file_index + position INTEGER, -- 1-based, matching the external reference table / m_FileID index + dependency_index INTEGER, -- references content_layout_serialized_files.file_index + PRIMARY KEY (serialized_file_index, position) +); diff --git a/Analyzer/Resources/ContentLayoutSerializedFiles.sql b/Analyzer/Resources/ContentLayoutSerializedFiles.sql new file mode 100644 index 0000000..3bf7b0b --- /dev/null +++ b/Analyzer/Resources/ContentLayoutSerializedFiles.sql @@ -0,0 +1,15 @@ +-- One row per entry in the layout's SerializedFiles array: the serialized files (.cf Content +-- Files) that the ContentDirectory build produced. file_index is the array index from the json +-- and is how the other content_layout tables reference a file. +-- serialized_file links to the core serialized_files table so layout data joins directly with +-- analyzed objects and references; it is NULL when the analyzed input did not include the build +-- content (e.g. a layout-only analyze) or for built-in entries. +CREATE TABLE IF NOT EXISTS content_layout_serialized_files +( + file_index INTEGER, + cfid TEXT, -- symbolic .cfid reference string (for built-ins: the built-in path) + is_builtin INTEGER, + content_hash TEXT, -- NULL for built-ins; the filename is content_hash || '.cf' + serialized_file INTEGER, -- references serialized_files.id, or NULL + PRIMARY KEY (file_index) +); diff --git a/Analyzer/Resources/ContentLayoutSourceAssets.sql b/Analyzer/Resources/ContentLayoutSourceAssets.sql new file mode 100644 index 0000000..b50458b --- /dev/null +++ b/Analyzer/Resources/ContentLayoutSourceAssets.sql @@ -0,0 +1,7 @@ +-- The source assets included in each serialized file (SourceAssets in the json). The same asset +-- path can appear in more than one file (e.g. an FBX split into multiple output files). +CREATE TABLE IF NOT EXISTS content_layout_source_assets +( + serialized_file_index INTEGER, -- references content_layout_serialized_files.file_index + asset_path TEXT +); diff --git a/Analyzer/Resources/ContentLayoutViews.sql b/Analyzer/Resources/ContentLayoutViews.sql new file mode 100644 index 0000000..47dd247 --- /dev/null +++ b/Analyzer/Resources/ContentLayoutViews.sql @@ -0,0 +1,64 @@ +-- Convenience views over the content_layout tables. Created together with the tables (only when +-- a ContentLayout.json is imported). See Documentation/contentlayout-database.md for the schema +-- reference. + +-- One row per layout serialized file with the derived filename, artifact size, and core-table link. +-- Built-in entries have no file on disk, so their path (cfid) is shown as the filename. +CREATE VIEW IF NOT EXISTS content_layout_serialized_files_view AS +SELECT f.file_index, f.cfid, f.is_builtin, + CASE WHEN f.is_builtin = 1 THEN f.cfid ELSE f.content_hash || '.cf' END AS filename, + ba.size, + f.serialized_file, + sf.archive +FROM content_layout_serialized_files f +LEFT JOIN content_layout_binary_artifacts ba ON ba.content_hash = f.content_hash AND ba.category = 'contentfile' +LEFT JOIN serialized_files sf ON sf.id = f.serialized_file; + +-- Source asset -> the file(s) it was built into. Built-in entries have no source assets, so no +-- filename fallback is needed here. +CREATE VIEW IF NOT EXISTS content_layout_source_assets_view AS +SELECT s.asset_path, f.file_index, f.content_hash || '.cf' AS filename, f.serialized_file +FROM content_layout_source_assets s +INNER JOIN content_layout_serialized_files f ON f.file_index = s.serialized_file_index; + +-- File-to-file dependency edges with names resolved on both sides (via the files view, so +-- dependencies on built-in entries show their path instead of a NULL filename). +CREATE VIEW IF NOT EXISTS content_layout_serialized_file_dependencies_view AS +SELECT d.serialized_file_index, src.filename, d.position, + d.dependency_index, dep.filename AS dependency_filename, dep.cfid AS dependency_cfid +FROM content_layout_serialized_file_dependencies d +INNER JOIN content_layout_serialized_files_view src ON src.file_index = d.serialized_file_index +INNER JOIN content_layout_serialized_files_view dep ON dep.file_index = d.dependency_index; + +-- Loadables resolved to their analyzed object (object columns are NULL in a layout-only database). +CREATE VIEW IF NOT EXISTS content_layout_loadable_objects_view AS +SELECT l.object_id_hash, l.guid, l.asset_path, l.lfid, l.is_root_asset, + f.content_hash || '.cf' AS filename, + o.id AS object, t.name AS type, o.name, o.size +FROM content_layout_loadable_objects l +LEFT JOIN content_layout_serialized_files f ON f.file_index = l.serialized_file_index +LEFT JOIN objects o ON o.serialized_file = f.serialized_file AND o.object_id = l.output_lfid +LEFT JOIN types t ON t.id = o.type; + +-- Artifacts with their derived on-disk filename (extension based on the category). +CREATE VIEW IF NOT EXISTS content_layout_binary_artifacts_view AS +SELECT artifact_index, content_hash, category, size, + content_hash || + CASE category + WHEN 'texture' THEN '.resS' WHEN 'mesh' THEN '.resS' + WHEN 'audio' THEN '.resource' WHEN 'video' THEN '.resource' + WHEN 'contentfile' THEN '.cf' WHEN 'manifest' THEN '.json' + ELSE '.' || category + END AS filename +FROM content_layout_binary_artifacts; + +-- The data files (.resS/.resource) each serialized file uses, derived from the artifact graph. +CREATE VIEW IF NOT EXISTS content_layout_data_files_view AS +SELECT f.file_index, f.content_hash || '.cf' AS filename, + ra.category, rav.filename AS data_filename, ra.size +FROM content_layout_serialized_files f +INNER JOIN content_layout_binary_artifacts ca ON ca.content_hash = f.content_hash AND ca.category = 'contentfile' +INNER JOIN content_layout_artifact_references r ON r.artifact_index = ca.artifact_index +INNER JOIN content_layout_binary_artifacts ra ON ra.artifact_index = r.referenced_artifact_index +INNER JOIN content_layout_binary_artifacts_view rav ON rav.artifact_index = ra.artifact_index +WHERE ra.category IN ('texture', 'mesh', 'audio', 'video'); diff --git a/Analyzer/SQLite/Commands/ContentLayout/AddContentLayout.cs b/Analyzer/SQLite/Commands/ContentLayout/AddContentLayout.cs new file mode 100644 index 0000000..7a56da9 --- /dev/null +++ b/Analyzer/SQLite/Commands/ContentLayout/AddContentLayout.cs @@ -0,0 +1,20 @@ +using System.Collections.Generic; +using Microsoft.Data.Sqlite; + +namespace UnityDataTools.Analyzer.SQLite.Commands.ContentLayout +{ + internal class AddContentLayout : AbstractCommand + { + protected override string TableName => "content_layout"; + + protected override string DDLSource => Properties.Resources.ContentLayout; + + protected override Dictionary Fields => new() + { + { "id", SqliteType.Integer }, + { "name", SqliteType.Text }, + { "version", SqliteType.Integer }, + { "build_manifest_hash", SqliteType.Text } + }; + } +} diff --git a/Analyzer/SQLite/Commands/ContentLayout/AddContentLayoutArtifactReference.cs b/Analyzer/SQLite/Commands/ContentLayout/AddContentLayoutArtifactReference.cs new file mode 100644 index 0000000..5bcd505 --- /dev/null +++ b/Analyzer/SQLite/Commands/ContentLayout/AddContentLayoutArtifactReference.cs @@ -0,0 +1,18 @@ +using System.Collections.Generic; +using Microsoft.Data.Sqlite; + +namespace UnityDataTools.Analyzer.SQLite.Commands.ContentLayout +{ + internal class AddContentLayoutArtifactReference : AbstractCommand + { + protected override string TableName => "content_layout_artifact_references"; + + protected override string DDLSource => Properties.Resources.ContentLayoutArtifactReferences; + + protected override Dictionary Fields => new() + { + { "artifact_index", SqliteType.Integer }, + { "referenced_artifact_index", SqliteType.Integer } + }; + } +} diff --git a/Analyzer/SQLite/Commands/ContentLayout/AddContentLayoutBinaryArtifact.cs b/Analyzer/SQLite/Commands/ContentLayout/AddContentLayoutBinaryArtifact.cs new file mode 100644 index 0000000..be04f1c --- /dev/null +++ b/Analyzer/SQLite/Commands/ContentLayout/AddContentLayoutBinaryArtifact.cs @@ -0,0 +1,20 @@ +using System.Collections.Generic; +using Microsoft.Data.Sqlite; + +namespace UnityDataTools.Analyzer.SQLite.Commands.ContentLayout +{ + internal class AddContentLayoutBinaryArtifact : AbstractCommand + { + protected override string TableName => "content_layout_binary_artifacts"; + + protected override string DDLSource => Properties.Resources.ContentLayoutBinaryArtifacts; + + protected override Dictionary Fields => new() + { + { "artifact_index", SqliteType.Integer }, + { "content_hash", SqliteType.Text }, + { "category", SqliteType.Text }, + { "size", SqliteType.Integer } + }; + } +} diff --git a/Analyzer/SQLite/Commands/ContentLayout/AddContentLayoutLoadableDependency.cs b/Analyzer/SQLite/Commands/ContentLayout/AddContentLayoutLoadableDependency.cs new file mode 100644 index 0000000..193587a --- /dev/null +++ b/Analyzer/SQLite/Commands/ContentLayout/AddContentLayoutLoadableDependency.cs @@ -0,0 +1,18 @@ +using System.Collections.Generic; +using Microsoft.Data.Sqlite; + +namespace UnityDataTools.Analyzer.SQLite.Commands.ContentLayout +{ + internal class AddContentLayoutLoadableDependency : AbstractCommand + { + protected override string TableName => "content_layout_loadable_dependencies"; + + protected override string DDLSource => Properties.Resources.ContentLayoutLoadableDependencies; + + protected override Dictionary Fields => new() + { + { "serialized_file_index", SqliteType.Integer }, + { "object_id_hash", SqliteType.Text } + }; + } +} diff --git a/Analyzer/SQLite/Commands/ContentLayout/AddContentLayoutLoadableObject.cs b/Analyzer/SQLite/Commands/ContentLayout/AddContentLayoutLoadableObject.cs new file mode 100644 index 0000000..044b61b --- /dev/null +++ b/Analyzer/SQLite/Commands/ContentLayout/AddContentLayoutLoadableObject.cs @@ -0,0 +1,24 @@ +using System.Collections.Generic; +using Microsoft.Data.Sqlite; + +namespace UnityDataTools.Analyzer.SQLite.Commands.ContentLayout +{ + internal class AddContentLayoutLoadableObject : AbstractCommand + { + protected override string TableName => "content_layout_loadable_objects"; + + protected override string DDLSource => Properties.Resources.ContentLayoutLoadableObjects; + + protected override Dictionary Fields => new() + { + { "object_id_hash", SqliteType.Text }, + { "guid", SqliteType.Text }, + { "asset_path", SqliteType.Text }, + { "lfid", SqliteType.Integer }, + { "identifier_type", SqliteType.Integer }, + { "serialized_file_index", SqliteType.Integer }, + { "output_lfid", SqliteType.Integer }, + { "is_root_asset", SqliteType.Integer } + }; + } +} diff --git a/Analyzer/SQLite/Commands/ContentLayout/AddContentLayoutLoadableScene.cs b/Analyzer/SQLite/Commands/ContentLayout/AddContentLayoutLoadableScene.cs new file mode 100644 index 0000000..5b8d0e1 --- /dev/null +++ b/Analyzer/SQLite/Commands/ContentLayout/AddContentLayoutLoadableScene.cs @@ -0,0 +1,19 @@ +using System.Collections.Generic; +using Microsoft.Data.Sqlite; + +namespace UnityDataTools.Analyzer.SQLite.Commands.ContentLayout +{ + internal class AddContentLayoutLoadableScene : AbstractCommand + { + protected override string TableName => "content_layout_loadable_scenes"; + + protected override string DDLSource => Properties.Resources.ContentLayoutLoadableScenes; + + protected override Dictionary Fields => new() + { + { "guid", SqliteType.Text }, + { "path", SqliteType.Text }, + { "serialized_file_index", SqliteType.Integer } + }; + } +} diff --git a/Analyzer/SQLite/Commands/ContentLayout/AddContentLayoutLoadableSceneDependency.cs b/Analyzer/SQLite/Commands/ContentLayout/AddContentLayoutLoadableSceneDependency.cs new file mode 100644 index 0000000..a9875b3 --- /dev/null +++ b/Analyzer/SQLite/Commands/ContentLayout/AddContentLayoutLoadableSceneDependency.cs @@ -0,0 +1,18 @@ +using System.Collections.Generic; +using Microsoft.Data.Sqlite; + +namespace UnityDataTools.Analyzer.SQLite.Commands.ContentLayout +{ + internal class AddContentLayoutLoadableSceneDependency : AbstractCommand + { + protected override string TableName => "content_layout_loadable_scene_dependencies"; + + protected override string DDLSource => Properties.Resources.ContentLayoutLoadableSceneDependencies; + + protected override Dictionary Fields => new() + { + { "serialized_file_index", SqliteType.Integer }, + { "scene_path", SqliteType.Text } + }; + } +} diff --git a/Analyzer/SQLite/Commands/ContentLayout/AddContentLayoutSerializedFile.cs b/Analyzer/SQLite/Commands/ContentLayout/AddContentLayoutSerializedFile.cs new file mode 100644 index 0000000..c2f639c --- /dev/null +++ b/Analyzer/SQLite/Commands/ContentLayout/AddContentLayoutSerializedFile.cs @@ -0,0 +1,21 @@ +using System.Collections.Generic; +using Microsoft.Data.Sqlite; + +namespace UnityDataTools.Analyzer.SQLite.Commands.ContentLayout +{ + internal class AddContentLayoutSerializedFile : AbstractCommand + { + protected override string TableName => "content_layout_serialized_files"; + + protected override string DDLSource => Properties.Resources.ContentLayoutSerializedFiles; + + protected override Dictionary Fields => new() + { + { "file_index", SqliteType.Integer }, + { "cfid", SqliteType.Text }, + { "is_builtin", SqliteType.Integer }, + { "content_hash", SqliteType.Text }, + { "serialized_file", SqliteType.Integer } + }; + } +} diff --git a/Analyzer/SQLite/Commands/ContentLayout/AddContentLayoutSerializedFileDependency.cs b/Analyzer/SQLite/Commands/ContentLayout/AddContentLayoutSerializedFileDependency.cs new file mode 100644 index 0000000..1bf7800 --- /dev/null +++ b/Analyzer/SQLite/Commands/ContentLayout/AddContentLayoutSerializedFileDependency.cs @@ -0,0 +1,19 @@ +using System.Collections.Generic; +using Microsoft.Data.Sqlite; + +namespace UnityDataTools.Analyzer.SQLite.Commands.ContentLayout +{ + internal class AddContentLayoutSerializedFileDependency : AbstractCommand + { + protected override string TableName => "content_layout_serialized_file_dependencies"; + + protected override string DDLSource => Properties.Resources.ContentLayoutSerializedFileDependencies; + + protected override Dictionary Fields => new() + { + { "serialized_file_index", SqliteType.Integer }, + { "position", SqliteType.Integer }, + { "dependency_index", SqliteType.Integer } + }; + } +} diff --git a/Analyzer/SQLite/Commands/ContentLayout/AddContentLayoutSourceAsset.cs b/Analyzer/SQLite/Commands/ContentLayout/AddContentLayoutSourceAsset.cs new file mode 100644 index 0000000..ffabc54 --- /dev/null +++ b/Analyzer/SQLite/Commands/ContentLayout/AddContentLayoutSourceAsset.cs @@ -0,0 +1,18 @@ +using System.Collections.Generic; +using Microsoft.Data.Sqlite; + +namespace UnityDataTools.Analyzer.SQLite.Commands.ContentLayout +{ + internal class AddContentLayoutSourceAsset : AbstractCommand + { + protected override string TableName => "content_layout_source_assets"; + + protected override string DDLSource => Properties.Resources.ContentLayoutSourceAssets; + + protected override Dictionary Fields => new() + { + { "serialized_file_index", SqliteType.Integer }, + { "asset_path", SqliteType.Text } + }; + } +} diff --git a/Analyzer/SQLite/Parsers/ContentLayoutParser.cs b/Analyzer/SQLite/Parsers/ContentLayoutParser.cs new file mode 100644 index 0000000..a70928a --- /dev/null +++ b/Analyzer/SQLite/Parsers/ContentLayoutParser.cs @@ -0,0 +1,90 @@ +using System; +using System.IO; +using Microsoft.Data.Sqlite; +using Newtonsoft.Json; +using UnityDataTools.Analyzer.SQLite.Handlers; +using UnityDataTools.Analyzer.SQLite.Writers; +using UnityDataTools.Models; + +namespace UnityDataTools.Analyzer.SQLite.Parsers +{ + // Imports the ContentLayout.json produced by BuildPipeline.BuildContentDirectory into the + // content_layout* tables (see Documentation/contentlayout.md for the file, and + // Documentation/contentlayout-database.md for the resulting schema). + public class ContentLayoutParser : ISQLiteFileParser + { + private ContentLayoutSQLWriter m_Writer; + private string m_ImportedLayout; + + public bool Verbose { get; set; } + public bool SkipReferences { get; set; } + public bool SkipCrc { get; set; } + + public void Init(SqliteConnection db) + { + m_Writer = new ContentLayoutSQLWriter(db); + } + + public bool CanParse(string filename) + { + // Unity always writes this exact filename into the build report directory, so unlike + // the Addressables build reports (whose filenames can embed timestamps) no content + // sniffing is needed. + return string.Equals(Path.GetFileName(filename), "ContentLayout.json", StringComparison.OrdinalIgnoreCase); + } + + public void Parse(string filename) + { + ContentLayout layout; + using (var reader = File.OpenText(filename)) + { + var serializer = new JsonSerializer(); + layout = (ContentLayout)serializer.Deserialize(reader, typeof(ContentLayout)); + } + + // The tool's failure summary only includes exception details in verbose mode, so + // report the reason for these expected failures directly. + if (layout == null) + { + Fail($"\"{filename}\" does not contain a ContentLayout."); + } + + if (layout.Version != ContentLayout.CurrentVersion) + { + Fail($"Unsupported ContentLayout.json version {layout.Version} (this version of UnityDataTool supports version {ContentLayout.CurrentVersion})."); + } + + if (m_ImportedLayout != null) + { + Fail($"Only a single ContentLayout.json can be analyzed (already imported \"{m_ImportedLayout}\")."); + } + + // Only create the content_layout tables when a layout is actually imported. + m_Writer.Init(); + m_Writer.WriteContentLayout(filename, layout); + m_ImportedLayout = filename; + } + + private static void Fail(string message) + { + Console.Error.WriteLine(); + Console.Error.WriteLine(message); + throw new Exception(message); + } + + // Called after all files are processed, so the analyzed .cf files all have their + // serialized_files rows and the layout entries can be linked to them. + public void FinalizeDatabase() + { + if (m_ImportedLayout != null) + { + m_Writer.LinkSerializedFiles(); + } + } + + public void Dispose() + { + m_Writer.Dispose(); + } + } +} diff --git a/Analyzer/SQLite/Writers/ContentLayoutSQLWriter.cs b/Analyzer/SQLite/Writers/ContentLayoutSQLWriter.cs new file mode 100644 index 0000000..4a89d49 --- /dev/null +++ b/Analyzer/SQLite/Writers/ContentLayoutSQLWriter.cs @@ -0,0 +1,260 @@ +using System; +using System.Collections.Generic; +using System.IO; +using Microsoft.Data.Sqlite; +using UnityDataTools.Analyzer.SQLite.Commands.ContentLayout; +using UnityDataTools.Models; + +namespace UnityDataTools.Analyzer.SQLite.Writers +{ + // Populates the content_layout* tables from a ContentLayout.json (see + // Documentation/contentlayout.md). The tables mirror the json structure, with two adjustments + // that make the data natural to query: the top-level RootAssets list is folded into the + // is_root_asset flag, and the json's sentinel values (-1 for "dropped from build", missing + // ContentHash for built-ins) are stored as NULL. + internal class ContentLayoutSQLWriter : IDisposable + { + private AddContentLayout m_AddContentLayout = new(); + private AddContentLayoutSerializedFile m_AddSerializedFile = new(); + private AddContentLayoutSourceAsset m_AddSourceAsset = new(); + private AddContentLayoutSerializedFileDependency m_AddSerializedFileDependency = new(); + private AddContentLayoutLoadableDependency m_AddLoadableDependency = new(); + private AddContentLayoutLoadableSceneDependency m_AddLoadableSceneDependency = new(); + private AddContentLayoutLoadableObject m_AddLoadableObject = new(); + private AddContentLayoutLoadableScene m_AddLoadableScene = new(); + private AddContentLayoutBinaryArtifact m_AddBinaryArtifact = new(); + private AddContentLayoutArtifactReference m_AddArtifactReference = new(); + + private bool m_Initialized; + private SqliteConnection m_Database; + + // The layout files that can have a .cf file on disk, kept for LinkSerializedFiles. + private List<(int Index, string ContentHash)> m_ImportedFiles = new(); + + public ContentLayoutSQLWriter(SqliteConnection database) + { + m_Database = database; + } + + // Creates the content_layout tables and views. Called lazily on the first import so that + // analyzing other content (AssetBundles, Player builds) doesn't create empty tables. + public void Init() + { + if (m_Initialized) + return; + + m_Initialized = true; + + m_AddContentLayout.CreateCommand(m_Database); + m_AddSerializedFile.CreateCommand(m_Database); + m_AddSourceAsset.CreateCommand(m_Database); + m_AddSerializedFileDependency.CreateCommand(m_Database); + m_AddLoadableDependency.CreateCommand(m_Database); + m_AddLoadableSceneDependency.CreateCommand(m_Database); + m_AddLoadableObject.CreateCommand(m_Database); + m_AddLoadableScene.CreateCommand(m_Database); + m_AddBinaryArtifact.CreateCommand(m_Database); + m_AddArtifactReference.CreateCommand(m_Database); + + ExecuteDDL(Properties.Resources.ContentLayoutViews); + } + + public void WriteContentLayout(string filename, ContentLayout layout) + { + using var transaction = m_Database.BeginTransaction(); + SetTransaction(transaction); + + try + { + m_AddContentLayout.SetValue("id", 0); + m_AddContentLayout.SetValue("name", filename); + m_AddContentLayout.SetValue("version", layout.Version); + m_AddContentLayout.SetValue("build_manifest_hash", layout.BuildManifestHash); + m_AddContentLayout.ExecuteNonQuery(); + + foreach (var file in layout.SerializedFiles) + { + m_AddSerializedFile.SetValue("file_index", file.Index); + m_AddSerializedFile.SetValue("cfid", file.ID); + m_AddSerializedFile.SetValue("is_builtin", file.IsBuiltIn ? 1 : 0); + m_AddSerializedFile.SetValue("content_hash", + string.IsNullOrEmpty(file.ContentHash) ? null : file.ContentHash); + // Filled in by LinkSerializedFiles when the analyzed input also contains the + // build content. + m_AddSerializedFile.SetValue("serialized_file", null); + m_AddSerializedFile.ExecuteNonQuery(); + + if (!string.IsNullOrEmpty(file.ContentHash)) + { + m_ImportedFiles.Add((file.Index, file.ContentHash)); + } + + // Empty arrays can be omitted from the json, leaving the fields null. + foreach (var assetPath in file.SourceAssets ?? []) + { + m_AddSourceAsset.SetValue("serialized_file_index", file.Index); + m_AddSourceAsset.SetValue("asset_path", assetPath); + m_AddSourceAsset.ExecuteNonQuery(); + } + + // The array order is significant (a PPtr's m_FileID resolves positionally), + // so it is preserved in the 1-based position column. + var dependencies = file.SerializedFileDependencies ?? []; + for (int i = 0; i < dependencies.Length; ++i) + { + m_AddSerializedFileDependency.SetValue("serialized_file_index", file.Index); + m_AddSerializedFileDependency.SetValue("position", i + 1); + m_AddSerializedFileDependency.SetValue("dependency_index", dependencies[i]); + m_AddSerializedFileDependency.ExecuteNonQuery(); + } + + foreach (var objectIdHash in file.LoadableDependencies ?? []) + { + m_AddLoadableDependency.SetValue("serialized_file_index", file.Index); + m_AddLoadableDependency.SetValue("object_id_hash", objectIdHash); + m_AddLoadableDependency.ExecuteNonQuery(); + } + + foreach (var scenePath in file.LoadableSceneDependencies ?? []) + { + m_AddLoadableSceneDependency.SetValue("serialized_file_index", file.Index); + m_AddLoadableSceneDependency.SetValue("scene_path", scenePath); + m_AddLoadableSceneDependency.ExecuteNonQuery(); + } + } + + var rootAssets = new HashSet(layout.RootAssets ?? []); + + foreach (var loadable in layout.LoadableObjectIds ?? []) + { + m_AddLoadableObject.SetValue("object_id_hash", loadable.ObjectIdHash); + m_AddLoadableObject.SetValue("guid", loadable.GUID); + m_AddLoadableObject.SetValue("asset_path", loadable.AssetPath); + m_AddLoadableObject.SetValue("lfid", loadable.LFID); + m_AddLoadableObject.SetValue("identifier_type", loadable.IdentifierType); + m_AddLoadableObject.SetValue("serialized_file_index", + loadable.SerializedFile < 0 ? null : loadable.SerializedFile); + m_AddLoadableObject.SetValue("output_lfid", loadable.OutputLFID); + m_AddLoadableObject.SetValue("is_root_asset", rootAssets.Contains(loadable.ObjectIdHash) ? 1 : 0); + m_AddLoadableObject.ExecuteNonQuery(); + } + + foreach (var scene in layout.LoadableSceneIds ?? []) + { + m_AddLoadableScene.SetValue("guid", scene.GUID); + m_AddLoadableScene.SetValue("path", scene.Path); + m_AddLoadableScene.SetValue("serialized_file_index", + scene.SerializedFile < 0 ? null : scene.SerializedFile); + m_AddLoadableScene.ExecuteNonQuery(); + } + + foreach (var artifact in layout.BinaryArtifacts ?? []) + { + m_AddBinaryArtifact.SetValue("artifact_index", artifact.Index); + m_AddBinaryArtifact.SetValue("content_hash", artifact.ContentHash); + m_AddBinaryArtifact.SetValue("category", artifact.Category); + m_AddBinaryArtifact.SetValue("size", artifact.Size); + m_AddBinaryArtifact.ExecuteNonQuery(); + + foreach (var referencedIndex in artifact.ArtifactReferences ?? []) + { + m_AddArtifactReference.SetValue("artifact_index", artifact.Index); + m_AddArtifactReference.SetValue("referenced_artifact_index", referencedIndex); + m_AddArtifactReference.ExecuteNonQuery(); + } + } + + transaction.Commit(); + } + catch (Exception) + { + transaction.Rollback(); + throw; + } + + // Indexes are created after the bulk insert so that a very large layout imports fast. + ExecuteDDL(Properties.Resources.ContentLayoutIndexes); + } + + // Fills in the serialized_file column, linking each layout entry to the serialized_files + // row of its analyzed .cf file. Called after all files are processed; entries whose file + // was not part of the analyzed input (e.g. a layout-only analyze) stay NULL. The match is + // on the file name (the content hash), ignoring any directory part that the analyze pass + // recorded in serialized_files.name. + public void LinkSerializedFiles() + { + var fileNameToId = new Dictionary(); + using (var select = m_Database.CreateCommand()) + { + select.CommandText = "SELECT id, name FROM serialized_files"; + using var reader = select.ExecuteReader(); + while (reader.Read()) + { + fileNameToId[Path.GetFileName(reader.GetString(1)).ToLowerInvariant()] = reader.GetInt32(0); + } + } + + using var transaction = m_Database.BeginTransaction(); + using var update = m_Database.CreateCommand(); + update.Transaction = transaction; + update.CommandText = "UPDATE content_layout_serialized_files SET serialized_file = @id WHERE file_index = @file_index"; + update.Parameters.Add("@id", SqliteType.Integer); + update.Parameters.Add("@file_index", SqliteType.Integer); + + try + { + foreach (var file in m_ImportedFiles) + { + if (fileNameToId.TryGetValue(file.ContentHash + ".cf", out var id)) + { + update.Parameters["@id"].Value = id; + update.Parameters["@file_index"].Value = file.Index; + update.ExecuteNonQuery(); + } + } + + transaction.Commit(); + } + catch (Exception) + { + transaction.Rollback(); + throw; + } + } + + private void SetTransaction(SqliteTransaction transaction) + { + m_AddContentLayout.SetTransaction(transaction); + m_AddSerializedFile.SetTransaction(transaction); + m_AddSourceAsset.SetTransaction(transaction); + m_AddSerializedFileDependency.SetTransaction(transaction); + m_AddLoadableDependency.SetTransaction(transaction); + m_AddLoadableSceneDependency.SetTransaction(transaction); + m_AddLoadableObject.SetTransaction(transaction); + m_AddLoadableScene.SetTransaction(transaction); + m_AddBinaryArtifact.SetTransaction(transaction); + m_AddArtifactReference.SetTransaction(transaction); + } + + private void ExecuteDDL(string sql) + { + using var command = m_Database.CreateCommand(); + command.CommandText = sql; + command.ExecuteNonQuery(); + } + + public void Dispose() + { + m_AddContentLayout.Dispose(); + m_AddSerializedFile.Dispose(); + m_AddSourceAsset.Dispose(); + m_AddSerializedFileDependency.Dispose(); + m_AddLoadableDependency.Dispose(); + m_AddLoadableSceneDependency.Dispose(); + m_AddLoadableObject.Dispose(); + m_AddLoadableScene.Dispose(); + m_AddBinaryArtifact.Dispose(); + m_AddArtifactReference.Dispose(); + } + } +} diff --git a/Documentation/analyzer.md b/Documentation/analyzer.md index 52fb897..099a2c6 100644 --- a/Documentation/analyzer.md +++ b/Documentation/analyzer.md @@ -282,6 +282,17 @@ SELECT * FROM dangling_refs_view WHERE source_id = 42; Because the view joins `refs`, it is not populated when analyze is run with `--skip-references` (in that mode neither `refs` nor `dangling_refs` are populated). +## ContentLayout (content_layout tables) + +When the analyzed input includes the `ContentLayout.json` of a ContentDirectory build (see +[contentlayout.md](contentlayout.md)), its content is imported into a set of `content_layout*` +tables and views: the source assets each file was built from, the dependencies between the files, +the loadable objects and scenes, and the size of every artifact, all connected to the analyzed +objects in the core tables. These tables are only created when a layout is actually imported. + +See [ContentLayout in the Analyze Database](contentlayout-database.md) for the tables, the views, +and how they join with the core tables. + ## BuildReport See [BuildReport.md](buildreport.md) for details of the tables and views related to analyzing BuildReport files. diff --git a/Documentation/contentlayout-database.md b/Documentation/contentlayout-database.md new file mode 100644 index 0000000..29f8c87 --- /dev/null +++ b/Documentation/contentlayout-database.md @@ -0,0 +1,92 @@ +# ContentLayout in the Analyze Database + +When the input of the [`analyze`](command-analyze.md) command includes the `ContentLayout.json` of a content directory build, its content is imported into the database tables and views documented on this page. This makes the layout of the build queryable with SQL — which source assets each file was built from, the dependencies between the files, the loadable objects and scenes, and the size of every artifact — and connects that information to the objects and references that `analyze` extracts from the build content itself. + +For what `ContentLayout.json` contains conceptually, see [ContentLayout.json](contentlayout.md); for the build output format it describes, see [Content Directory Format](contentdirectory-format.md). + +## General notes + +* The `content_layout*` tables and views are only created when a `ContentLayout.json` is actually part of the analyzed input, so databases produced from AssetBundles or Player builds are not cluttered with empty tables. +* A single layout per database is supported. Additional `ContentLayout.json` files on the input are reported as failed. +* The tables mirror the json structure. Entries that cross-reference each other by array index in the json do the same in the database: `file_index` and `artifact_index` columns hold the json array indexes. +* Two adjustments are made so the data is natural to query: + * The json's sentinel values are stored as SQL `NULL`: a `SerializedFile` value of `-1` (an object dropped from the build) and the missing `ContentHash` of built-in entries. + * The top-level `RootAssets` list is folded into the `is_root_asset` flag of `content_layout_loadable_objects`. +* The `.sql` files in [`Analyzer/Resources`](../Analyzer/Resources) (`ContentLayout*.sql`) are the authoritative reference for the exact columns, in the same way [`ContentLayout.cs`](../UnityDataModels/ContentLayout.cs) is for the json schema. + +## Tables + +### content_layout + +One row identifying the imported layout: `name` (the path of the imported file), `version` (the json schema version) and `build_manifest_hash`. + +### content_layout_serialized_files + +One row per serialized file (`.cf` Content File) of the build, keyed by `file_index`. Records the symbolic `cfid` reference string, the `is_builtin` flag, and the `content_hash` that gives the filename (`content_hash || '.cf'`). + +The `serialized_file` column references the core `serialized_files` table when the build content is part of the analyzed input, connecting the layout to the analyzed objects (see [below](#analyzing-the-layout-with-or-without-the-build-content)). + +### content_layout_source_assets + +The source assets included in each serialized file, one row per (`serialized_file_index`, `asset_path`) pair. The same asset path can appear in more than one file (for example, a single FBX split into multiple output files). + +### content_layout_serialized_file_dependencies + +File-to-file dependency edges: the other serialized files that must be loaded before this one (`serialized_file_index` → `dependency_index`). The 1-based `position` column preserves the json array order, which is significant: a PPtr's `m_FileID` inside the file resolves positionally through this list (see [Content Directory Format](contentdirectory-format.md)). + +### content_layout_loadable_dependencies / content_layout_loadable_scene_dependencies + +The loadables (`object_id_hash`) and scenes (`scene_path`) referenced from each serialized file. + +### content_layout_loadable_objects + +The objects that can be loaded on demand, keyed by `object_id_hash`. Each row records where the object came from in the source project (`guid`, `asset_path`, `lfid`, `identifier_type`) and where it lives in the built content (`serialized_file_index`, `output_lfid`). `is_root_asset` marks the root assets the build was made from. + +### content_layout_loadable_scenes + +The scenes exposed as loadable in the build: `guid`, `path` and the `serialized_file_index` of the file containing the scene. + +### content_layout_binary_artifacts + +Every artifact of the build output, keyed by `artifact_index`: the serialized files themselves (`category = 'contentfile'`), the streamed texture/mesh data (`'texture'`, `'mesh'` → `.resS` files), the audio/video data (`'audio'`, `'video'` → `.resource` files) and the manifest (`'manifest'`). This is the standard place to find artifact sizes, including the sizes of the serialized files, which no core table records. + +### content_layout_artifact_references + +Direct references between artifacts (`artifact_index` → `referenced_artifact_index`), e.g. a serialized file referencing its data files. References that go through a loadable are not included, and the graph is never cyclical. References to other serialized files are not recorded here either — those are in `content_layout_serialized_file_dependencies`. + +## Views + +| View | Description | +|------|-------------| +| `content_layout_serialized_files_view` | One row per layout serialized file with the derived filename, its artifact size, and the core-table link (`serialized_file`, `archive`). Built-in entries show their path as the filename. | +| `content_layout_source_assets_view` | Source asset → the file(s) it was built into. | +| `content_layout_serialized_file_dependencies_view` | The dependency edges with filenames resolved on both sides. | +| `content_layout_loadable_objects_view` | The loadables resolved to their analyzed object (`object`, `type`, `name`, `size`). | +| `content_layout_binary_artifacts_view` | The artifacts with their derived on-disk filename (content hash + category-based extension). | +| `content_layout_data_files_view` | The `.resS`/`.resource` data files each serialized file uses, derived from the artifact graph. | + +```sql +-- which files was this source asset built into? +SELECT * FROM content_layout_source_assets_view WHERE asset_path = 'Assets/Textures/GreenStatic.png'; + +-- build size by artifact category +SELECT category, COUNT(*) AS count, SUM(size) AS bytes +FROM content_layout_binary_artifacts GROUP BY category ORDER BY bytes DESC; +``` + +## Analyzing the layout with or without the build content + +A `ContentLayout.json` can be analyzed on its own — useful for running SQL queries against a large layout — or together with the build output it describes. The layout tables themselves are identical in both cases; what differs is the connection to the core tables: + +* When the build content is part of the analyzed input, each `content_layout_serialized_files` row is linked to its analyzed file through the `serialized_file` column, and the views resolve across that link (e.g. `content_layout_loadable_objects_view` shows the object, type, name and size of each loadable). +* In a layout-only analyze there is nothing to link to, so expect NULL in `content_layout_serialized_files.serialized_file`, in the `archive` column of `content_layout_serialized_files_view`, and in the `object`, `type`, `name` and `size` columns of `content_layout_loadable_objects_view`. +* Built-in entries (`is_builtin = 1`) always have a NULL `content_hash` and `serialized_file` — they are not files produced by the build. + +## Related documentation + +| Topic | Description | +|-------|-------------| +| [ContentLayout.json](contentlayout.md) | What the file contains and its json schema. | +| [Content Directory Format](contentdirectory-format.md) | Content directory builds and inspecting them with UnityDataTool. | +| [`analyze` command](command-analyze.md) | The command that imports the layout. | +| [Analyzer](analyzer.md) | The core database schema (objects, serialized files, references). | diff --git a/Documentation/contentlayout.md b/Documentation/contentlayout.md index 996338a..30ce27a 100644 --- a/Documentation/contentlayout.md +++ b/Documentation/contentlayout.md @@ -67,6 +67,7 @@ The schema is subject to change. The `Version` field records the schema version | Topic | Description | |-------|-------------| +| [ContentLayout in the Analyze Database](contentlayout-database.md) | How the [`analyze` command](command-analyze.md) imports this file into queryable database tables. | | [Content Directory Format](contentdirectory-format.md) | Content directory builds and inspecting them with UnityDataTool. | | [Build report and build history](https://docs.unity3d.com/6000.6/Documentation/Manual/build-reporting.html) | The build report directory and the files in it (Unity Manual). | | [BuildReport Support](buildreport.md) | Analyzing Unity build report files with UnityDataTool. | diff --git a/UnityDataTool.Tests/AnalyzeContentLayoutTests.cs b/UnityDataTool.Tests/AnalyzeContentLayoutTests.cs new file mode 100644 index 0000000..97c4429 --- /dev/null +++ b/UnityDataTool.Tests/AnalyzeContentLayoutTests.cs @@ -0,0 +1,256 @@ +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Data.Sqlite; +using NUnit.Framework; + +namespace UnityDataTools.UnityDataTool.Tests; + +#pragma warning disable NUnit2005, NUnit2006 + +// Tests the import of ContentLayout.json into the content_layout* tables (issue #99). Runs +// against the ContentLayout.json of the LeadingEdge ContentDirectory reference build, whose +// content is well-known (see UnityProjects/LeadingEdge/Assets/Editor/BuildContentDirectory.cs): +// 14 serialized files (1 built-in), 3 loadable objects with ContentDirectoryRoot as the single +// root asset, 2 loadable scenes, and 18 binary artifacts. +public class AnalyzeContentLayoutTests +{ + private string m_TestOutputFolder; + private string m_ContentLayoutPath; + + [OneTimeSetUp] + public void OneTimeSetup() + { + m_TestOutputFolder = Path.Combine(TestContext.CurrentContext.TestDirectory, "content_layout_test_folder"); + m_ContentLayoutPath = Path.Combine(TestContext.CurrentContext.TestDirectory, + "Data", "LeadingEdgeBuilds", "BuildReport-ContentDirectory", "ContentLayout.json"); + Directory.CreateDirectory(m_TestOutputFolder); + Directory.SetCurrentDirectory(m_TestOutputFolder); + } + + [TearDown] + public void Teardown() + { + SqliteConnection.ClearAllPools(); + var outputFolder = new DirectoryInfo(m_TestOutputFolder); + outputFolder.EnumerateFiles().ToList().ForEach(f => f.Delete()); + outputFolder.EnumerateDirectories().ToList().ForEach(d => d.Delete(true)); + } + + [Test] + public async Task Analyze_ContentLayoutOnly_ImportsLayoutTables() + { + var databasePath = SQLTestHelper.GetDatabasePath(m_TestOutputFolder); + + Assert.AreEqual(0, await Program.Main(new string[] { "analyze", m_ContentLayoutPath, "-o", databasePath })); + using var db = SQLTestHelper.OpenDatabase(databasePath); + + // Header row identifying the imported layout. + SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM content_layout", 1, + "a single layout should be imported"); + SQLTestHelper.AssertQueryInt(db, "SELECT version FROM content_layout", 2, + "the layout schema version should be recorded"); + SQLTestHelper.AssertQueryString(db, "SELECT build_manifest_hash FROM content_layout", + "baff06b928d147276f2245dd3b19216a", "the BuildManifestHash should be recorded"); + + // Row counts of each table, matching the json content. + SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM content_layout_serialized_files", 14, + "one row per SerializedFiles entry"); + SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM content_layout_source_assets", 16, + "one row per source asset per file"); + SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM content_layout_serialized_file_dependencies", 16, + "one row per file dependency"); + SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM content_layout_loadable_dependencies", 3, + "one row per loadable dependency"); + SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM content_layout_loadable_scene_dependencies", 2, + "one row per loadable scene dependency"); + SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM content_layout_loadable_objects", 3, + "one row per LoadableObjectIds entry"); + SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM content_layout_loadable_scenes", 2, + "one row per LoadableSceneIds entry"); + SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM content_layout_binary_artifacts", 18, + "one row per BinaryArtifacts entry"); + SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM content_layout_artifact_references", 17, + "one row per artifact reference"); + + // The built-in entry keeps its human-readable ID and has no content hash. + SQLTestHelper.AssertQueryString(db, + "SELECT cfid FROM content_layout_serialized_files WHERE file_index = 0", + "Library/unity default resources", "the built-in entry should be at index 0"); + SQLTestHelper.AssertQueryInt(db, + "SELECT COUNT(*) FROM content_layout_serialized_files WHERE is_builtin = 1", 1, + "the reference build has a single built-in entry"); + SQLTestHelper.AssertQueryInt(db, + "SELECT COUNT(*) FROM content_layout_serialized_files WHERE is_builtin = 1 AND content_hash IS NOT NULL", 0, + "built-in entries have no content hash"); + + // The ContentDirectoryRoot file and its dependencies, preserving the json array order + // (json SerializedFileDependencies for index 5: [8, 4, 11, 13, 7]). + SQLTestHelper.AssertQueryString(db, + "SELECT cfid FROM content_layout_serialized_files WHERE file_index = 5", + "52b43dad178849b42ac753005736e7bb.cfid", "cfid of the ContentDirectoryRoot file"); + SQLTestHelper.AssertQueryString(db, + "SELECT content_hash FROM content_layout_serialized_files WHERE file_index = 5", + "c0152db4dd710be51b2decb997325f34", "content hash of the ContentDirectoryRoot file"); + SQLTestHelper.AssertQueryString(db, + @"SELECT GROUP_CONCAT(dependency_index) FROM ( + SELECT dependency_index FROM content_layout_serialized_file_dependencies + WHERE serialized_file_index = 5 ORDER BY position)", + "8,4,11,13,7", "dependency order must match the json array order"); + SQLTestHelper.AssertQueryString(db, + "SELECT asset_path FROM content_layout_source_assets WHERE serialized_file_index = 5", + "Assets/ScriptableObjects/ContentDirectoryRoot.asset", "source asset of the root file"); + + // RootAssets is folded into the is_root_asset flag. + SQLTestHelper.AssertQueryString(db, + "SELECT asset_path FROM content_layout_loadable_objects WHERE is_root_asset = 1", + "Assets/ScriptableObjects/ContentDirectoryRoot.asset", + "ContentDirectoryRoot is the only root asset"); + + // The core-table link is only populated when the build content is analyzed too. + SQLTestHelper.AssertQueryInt(db, + "SELECT COUNT(*) FROM content_layout_serialized_files WHERE serialized_file IS NOT NULL", 0, + "a layout-only analyze cannot link to analyzed serialized files"); + + SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM content_layout_binary_artifacts WHERE category = 'contentfile'", + 13, "one contentfile artifact per non-built-in serialized file"); + SQLTestHelper.AssertQueryString(db, + "SELECT filename FROM content_layout_binary_artifacts_view WHERE category = 'manifest'", + "baff06b928d147276f2245dd3b19216a.json", "the artifact filename is derived from the category"); + + // Views are created with the tables and their joins produce the expected rows. + SQLTestHelper.AssertViewExists(db, "content_layout_serialized_files_view"); + SQLTestHelper.AssertViewExists(db, "content_layout_source_assets_view"); + SQLTestHelper.AssertViewExists(db, "content_layout_serialized_file_dependencies_view"); + SQLTestHelper.AssertViewExists(db, "content_layout_loadable_objects_view"); + SQLTestHelper.AssertViewExists(db, "content_layout_binary_artifacts_view"); + SQLTestHelper.AssertViewExists(db, "content_layout_data_files_view"); + + SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM content_layout_data_files_view", 4, + "the reference build has 2 .resS and 2 .resource data files"); + SQLTestHelper.AssertQueryString(db, + "SELECT filename FROM content_layout_serialized_files_view WHERE is_builtin = 1", + "Library/unity default resources", "built-in entries show their path as the filename"); + SQLTestHelper.AssertQueryString(db, + @"SELECT dependency_filename FROM content_layout_serialized_file_dependencies_view + WHERE serialized_file_index = 2 AND position = 1", + "Library/unity default resources", "dependencies on built-in entries resolve to their path"); + SQLTestHelper.AssertQueryString(db, + @"SELECT filename FROM content_layout_loadable_objects_view + WHERE asset_path = 'Assets/ScriptableObjects/ContentDirectoryRoot.asset'", + "c0152db4dd710be51b2decb997325f34.cf", "the loadable view resolves the containing file"); + SQLTestHelper.AssertQueryString(db, + @"SELECT dependency_filename FROM content_layout_serialized_file_dependencies_view + WHERE serialized_file_index = 5 AND position = 1", + "86d71ff2bb38e064697257d35d6421b8.cf", "the dependencies view resolves target filenames"); + } + + [Test] + public async Task Analyze_ContentDirectoryWithLayout_LinksLayoutToAnalyzedContent() + { + var contentDirectory = Path.Combine(TestContext.CurrentContext.TestDirectory, + "Data", "LeadingEdgeBuilds", "ContentDirectory"); + var databasePath = SQLTestHelper.GetDatabasePath(m_TestOutputFolder); + + Assert.AreEqual(0, await Program.Main(new string[] { "analyze", m_ContentLayoutPath, contentDirectory, "-o", databasePath })); + using var db = SQLTestHelper.OpenDatabase(databasePath); + + // Every non-built-in layout entry links to the serialized_files row of its analyzed file. + SQLTestHelper.AssertQueryInt(db, + "SELECT COUNT(*) FROM content_layout_serialized_files WHERE is_builtin = 0 AND serialized_file IS NULL", 0, + "all analyzed files should be linked"); + SQLTestHelper.AssertQueryInt(db, + "SELECT COUNT(*) FROM content_layout_serialized_files WHERE is_builtin = 1 AND serialized_file IS NOT NULL", 0, + "built-in entries have no file to link to"); + SQLTestHelper.AssertQueryInt(db, + @"SELECT COUNT(*) FROM content_layout_serialized_files f + WHERE f.serialized_file IS NOT NULL + AND NOT EXISTS (SELECT 1 FROM objects o WHERE o.serialized_file = f.serialized_file)", 0, + "every linked file should have analyzed objects"); + + // The loadables resolve to their analyzed objects through the link. + SQLTestHelper.AssertQueryInt(db, + "SELECT COUNT(*) FROM content_layout_loadable_objects_view WHERE object IS NULL", 0, + "every loadable should resolve to an analyzed object"); + SQLTestHelper.AssertQueryString(db, + @"SELECT name FROM content_layout_loadable_objects_view + WHERE asset_path = 'Assets/ScriptableObjects/ContentDirectoryRoot.asset'", + "ContentDirectoryRoot", "the root loadable resolves to the root ScriptableObject"); + SQLTestHelper.AssertQueryString(db, + @"SELECT type FROM content_layout_loadable_objects_view + WHERE asset_path = 'Assets/ScriptableObjects/ContentDirectoryRoot.asset'", + "MonoBehaviour", "ScriptableObjects are serialized as MonoBehaviour"); + } + + [Test] + public async Task Analyze_WithoutContentLayout_DoesNotCreateLayoutTables() + { + var bundlePath = Path.Combine(TestContext.CurrentContext.TestDirectory, + "Data", "LeadingEdgeBuilds", "AssetBundles", "assetbundleroot"); + var databasePath = SQLTestHelper.GetDatabasePath(m_TestOutputFolder); + + Assert.AreEqual(0, await Program.Main(new string[] { "analyze", bundlePath, "-o", databasePath })); + using var db = SQLTestHelper.OpenDatabase(databasePath); + + SQLTestHelper.AssertQueryInt(db, + "SELECT COUNT(*) FROM sqlite_master WHERE name LIKE 'content_layout%'", 0, + "the content_layout tables are only created when a ContentLayout.json is imported"); + } + + [Test] + public async Task Analyze_UnsupportedLayoutVersion_ImportsNothing() + { + var layoutFolder = Path.Combine(m_TestOutputFolder, "future_version"); + Directory.CreateDirectory(layoutFolder); + File.WriteAllText(Path.Combine(layoutFolder, "ContentLayout.json"), "{\"Version\": 99}"); + var databasePath = SQLTestHelper.GetDatabasePath(m_TestOutputFolder); + + // Analyze reports the file as failed but the run itself still completes. + Assert.AreEqual(0, await Program.Main(new string[] { "analyze", layoutFolder, "-o", databasePath })); + using var db = SQLTestHelper.OpenDatabase(databasePath); + + SQLTestHelper.AssertQueryInt(db, + "SELECT COUNT(*) FROM sqlite_master WHERE name LIKE 'content_layout%'", 0, + "an unsupported layout version must not be imported"); + } + + [Test] + public async Task Analyze_ContentLayoutWithoutContent_ImportsNothing() + { + var layoutFolder = Path.Combine(m_TestOutputFolder, "null_layout"); + Directory.CreateDirectory(layoutFolder); + File.WriteAllText(Path.Combine(layoutFolder, "ContentLayout.json"), "null"); + var databasePath = SQLTestHelper.GetDatabasePath(m_TestOutputFolder); + + Assert.AreEqual(0, await Program.Main(new string[] { "analyze", layoutFolder, "-o", databasePath })); + using var db = SQLTestHelper.OpenDatabase(databasePath); + + SQLTestHelper.AssertQueryInt(db, + "SELECT COUNT(*) FROM sqlite_master WHERE name LIKE 'content_layout%'", 0, + "a json file without a ContentLayout must not be imported"); + } + + [Test] + public async Task Analyze_MultipleContentLayouts_ImportsOnlyTheFirst() + { + // Two directories each containing a (identical) ContentLayout.json. Only a single layout + // per database is supported; the second one is reported as failed. + var folderA = Path.Combine(m_TestOutputFolder, "layout_a"); + var folderB = Path.Combine(m_TestOutputFolder, "layout_b"); + Directory.CreateDirectory(folderA); + Directory.CreateDirectory(folderB); + File.Copy(m_ContentLayoutPath, Path.Combine(folderA, "ContentLayout.json")); + File.Copy(m_ContentLayoutPath, Path.Combine(folderB, "ContentLayout.json")); + var databasePath = SQLTestHelper.GetDatabasePath(m_TestOutputFolder); + + Assert.AreEqual(0, await Program.Main(new string[] { "analyze", folderA, folderB, "-o", databasePath })); + using var db = SQLTestHelper.OpenDatabase(databasePath); + + SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM content_layout", 1, + "only a single layout should be imported"); + Assert.IsTrue(SQLTestHelper.QueryString(db, "SELECT name FROM content_layout").Contains("layout_a"), + "the first layout on the input should be the imported one"); + SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM content_layout_serialized_files", 14, + "the imported layout content should be intact"); + } +}