diff --git a/Analyzer/AnalyzerTool.cs b/Analyzer/AnalyzerTool.cs index 5790f66..80ec988 100644 --- a/Analyzer/AnalyzerTool.cs +++ b/Analyzer/AnalyzerTool.cs @@ -2,9 +2,12 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.Linq; +using Newtonsoft.Json; using UnityDataTools.Analyzer.SQLite.Handlers; using UnityDataTools.Analyzer.SQLite.Parsers; using UnityDataTools.Analyzer.SQLite.Writers; +using UnityDataTools.Analyzer.Util; using UnityDataTools.FileSystem; using UnityDataTools.Models; @@ -14,12 +17,23 @@ public class AnalyzerTool { AnalyzeOptions m_Options; - public List parsers = new List() + // Shared between the ContentLayout import and the serialized-file analysis: both must agree + // on the id assigned to each serialized file name, and the layout's dependency information + // is what resolves the external references of ContentDirectory files (issue #99). + private IdProvider m_SerializedFileIdProvider = new(); + private ContentFileDependencyMap m_ContentFileDependencies = new(); + + public List parsers; + + public AnalyzerTool() { - new ContentLayoutParser(), - new AddressablesBuildLayoutParser(), - new SerializedFileParser(), - }; + parsers = new List() + { + new ContentLayoutParser(m_SerializedFileIdProvider, m_ContentFileDependencies), + new AddressablesBuildLayoutParser(), + new SerializedFileParser(m_SerializedFileIdProvider, m_ContentFileDependencies), + }; + } public class AnalyzeOptions { @@ -38,6 +52,15 @@ public int Analyze(AnalyzeOptions options) { m_Options = options; + var files = CollectFiles(); + + // Validate the ContentDirectory-related inputs before creating the database, so an + // invalid combination fails without leaving a partial database behind. + if (!PrepareContentDirectoryInputs(files)) + { + return 1; + } + using SQLiteWriter writer = new(m_Options.DatabaseName); try @@ -61,8 +84,6 @@ public int Analyze(AnalyzeOptions options) var timer = new Stopwatch(); timer.Start(); - var files = CollectFiles(); - int countFailures = 0; int countSuccess = 0; int countIgnored = 0; @@ -152,6 +173,149 @@ public int Analyze(AnalyzeOptions options) return 0; } + // Validates the ContentDirectory-related inputs and prepares the file list (issue #99): + // enforces that a single build is analyzed, selects the ContentLayout.json whose + // BuildManifestHash matches that build (dropping any others), and moves it to the front of + // the list so it is imported before the content files whose references it resolves. Returns + // false, after printing an error, when the input combination is invalid. + bool PrepareContentDirectoryInputs(List<(string FullPath, string DisplayRoot)> files) + { + const string hashFileName = "BuildManifestHash.txt"; + + var layoutCandidates = files.Where(f => ContentLayoutParser.IsContentLayoutFile(f.FullPath)).ToList(); + var hashFiles = files + .Where(f => string.Equals(Path.GetFileName(f.FullPath), hashFileName, StringComparison.OrdinalIgnoreCase)) + .Select(f => f.FullPath) + .ToList(); + + // ContentDirectory output is recognized by its .cf content files, or by its archive when + // built compressed. The BuildManifestHash.txt identifying the build sits next to them; it + // is not always on the input (e.g. when specific files are passed), so pick it up from + // the directories containing the content. + var contentDirectories = files + .Where(f => HasExtension(f.FullPath, ".cf") || HasExtension(f.FullPath, ".archive")) + .Select(f => Path.GetDirectoryName(Path.GetFullPath(f.FullPath))) + .Distinct(StringComparer.OrdinalIgnoreCase); + + if (hashFiles.Count == 0) + { + hashFiles.AddRange(contentDirectories + .Select(dir => Path.Combine(dir, hashFileName)) + .Where(File.Exists)); + } + + List buildHashes; + try + { + buildHashes = hashFiles.Select(f => File.ReadAllText(f).Trim()).Distinct().ToList(); + } + catch (Exception e) + { + Console.Error.WriteLine($"Error reading {hashFileName}: {e.Message}"); + return false; + } + + if (buildHashes.Count > 1) + { + Console.Error.WriteLine("The input contains more than one ContentDirectory build (different BuildManifestHash.txt values). Analyze a single build at a time."); + return false; + } + + var buildHash = buildHashes.Count == 1 ? buildHashes[0] : null; + var hasContentDirectory = buildHash != null || files.Any(f => HasExtension(f.FullPath, ".cf")); + + if (layoutCandidates.Count == 0) + { + if (hasContentDirectory) + { + Console.Error.WriteLine( + "Warning: analyzing ContentDirectory output without its ContentLayout.json. The analysis will be incomplete: " + + "references between content files cannot be resolved (they will appear in dangling_refs) and source asset " + + "information is unavailable. Re-run with the build's ContentLayout.json (found in its build report folder) " + + "included in the input paths."); + } + + return true; + } + + (string FullPath, string DisplayRoot) selected; + + if (hasContentDirectory) + { + if (buildHash == null) + { + Console.Error.WriteLine("A ContentLayout.json is in the input but no BuildManifestHash.txt was found for the ContentDirectory content, so the layout cannot be validated against the build."); + return false; + } + + // The hash match guarantees the layout describes exactly this build; a stale or + // unrelated layout would silently produce misleading results. + selected = layoutCandidates.FirstOrDefault(c => TryReadBuildManifestHash(c.FullPath) == buildHash); + + if (selected.FullPath == null) + { + Console.Error.WriteLine($"No ContentLayout.json in the input matches the analyzed build (BuildManifestHash {buildHash}). Include the layout from the build report folder of this build."); + return false; + } + } + else if (layoutCandidates.Count == 1) + { + // A layout without its build content is a valid input (e.g. to query a large layout). + selected = layoutCandidates[0]; + } + else + { + Console.Error.WriteLine("The input contains multiple ContentLayout.json files but no ContentDirectory build to match them against. Only a single layout can be analyzed."); + return false; + } + + foreach (var candidate in layoutCandidates) + { + if (candidate != selected) + { + Console.Error.WriteLine($"Ignoring \"{candidate.FullPath}\": its BuildManifestHash does not match the analyzed build."); + files.Remove(candidate); + } + } + + // Import the layout before the content files it describes, so their references can be + // resolved through it. + files.Remove(selected); + files.Insert(0, selected); + + return true; + } + + static bool HasExtension(string path, string extension) + { + return string.Equals(Path.GetExtension(path), extension, StringComparison.OrdinalIgnoreCase); + } + + // Reads the top-level BuildManifestHash of a ContentLayout.json without parsing the whole + // file (layouts of large builds are big; the hash is one of the first properties). Returns + // null when the value cannot be found or the file is not valid json. + static string TryReadBuildManifestHash(string path) + { + try + { + using var reader = new JsonTextReader(File.OpenText(path)); + + for (int i = 0; i < 64 && reader.Read(); ++i) + { + if (reader.TokenType == JsonToken.PropertyName && reader.Depth == 1 && + "BuildManifestHash".Equals(reader.Value)) + { + return reader.ReadAsString(); + } + } + } + catch (Exception) + { + } + + return null; + } + // Expands the input paths into the concrete files to analyze. Each result pairs the file with the // root used to render its relative path in progress/error messages: the scanned directory for files // found by scanning, or the file's own directory for explicitly-named files. Duplicates reached via diff --git a/Analyzer/SQLite/Parsers/ContentLayoutParser.cs b/Analyzer/SQLite/Parsers/ContentLayoutParser.cs index a70928a..2add87f 100644 --- a/Analyzer/SQLite/Parsers/ContentLayoutParser.cs +++ b/Analyzer/SQLite/Parsers/ContentLayoutParser.cs @@ -4,6 +4,7 @@ using Newtonsoft.Json; using UnityDataTools.Analyzer.SQLite.Handlers; using UnityDataTools.Analyzer.SQLite.Writers; +using UnityDataTools.Analyzer.Util; using UnityDataTools.Models; namespace UnityDataTools.Analyzer.SQLite.Parsers @@ -15,24 +16,37 @@ public class ContentLayoutParser : ISQLiteFileParser { private ContentLayoutSQLWriter m_Writer; private string m_ImportedLayout; + private IdProvider m_SerializedFileIdProvider; + private ContentFileDependencyMap m_ContentFileDependencies; public bool Verbose { get; set; } public bool SkipReferences { get; set; } public bool SkipCrc { get; set; } + public ContentLayoutParser(IdProvider serializedFileIdProvider, ContentFileDependencyMap contentFileDependencies) + { + m_SerializedFileIdProvider = serializedFileIdProvider; + m_ContentFileDependencies = contentFileDependencies; + } + public void Init(SqliteConnection db) { - m_Writer = new ContentLayoutSQLWriter(db); + m_Writer = new ContentLayoutSQLWriter(db, m_SerializedFileIdProvider, m_ContentFileDependencies); } - public bool CanParse(string filename) + // Unity always writes this exact filename into the build report directory, so unlike the + // Addressables build reports (whose filenames can embed timestamps) no content sniffing + // is needed. + public static bool IsContentLayoutFile(string filename) { - // Unity always writes this exact filename into the build report directory, so unlike - // the Addressables build reports (whose filenames can embed timestamps) no content - // sniffing is needed. return string.Equals(Path.GetFileName(filename), "ContentLayout.json", StringComparison.OrdinalIgnoreCase); } + public bool CanParse(string filename) + { + return IsContentLayoutFile(filename); + } + public void Parse(string filename) { ContentLayout layout; diff --git a/Analyzer/SQLite/Parsers/SerializedFileParser.cs b/Analyzer/SQLite/Parsers/SerializedFileParser.cs index ec66c8b..b937d8e 100644 --- a/Analyzer/SQLite/Parsers/SerializedFileParser.cs +++ b/Analyzer/SQLite/Parsers/SerializedFileParser.cs @@ -12,11 +12,19 @@ namespace UnityDataTools.Analyzer.SQLite.Parsers public class SerializedFileParser : ISQLiteFileParser { private SerializedFileSQLiteWriter m_Writer; + private Util.IdProvider m_SerializedFileIdProvider; + private Util.ContentFileDependencyMap m_ContentFileDependencies; public bool Verbose { get; set; } public bool SkipReferences { get; set; } public bool SkipCrc { get; set; } + public SerializedFileParser(Util.IdProvider serializedFileIdProvider, Util.ContentFileDependencyMap contentFileDependencies) + { + m_SerializedFileIdProvider = serializedFileIdProvider; + m_ContentFileDependencies = contentFileDependencies; + } + public bool CanParse(string filename) { // First check if the file is in the ignore list (by extension or filename) @@ -43,7 +51,8 @@ public void FinalizeDatabase() public void Init(SqliteConnection db) { - m_Writer = new SerializedFileSQLiteWriter(db, SkipReferences, SkipCrc); + m_Writer = new SerializedFileSQLiteWriter(db, SkipReferences, SkipCrc, + m_SerializedFileIdProvider, m_ContentFileDependencies); } public void Parse(string filename) diff --git a/Analyzer/SQLite/Writers/ContentLayoutSQLWriter.cs b/Analyzer/SQLite/Writers/ContentLayoutSQLWriter.cs index 4a89d49..263a4e8 100644 --- a/Analyzer/SQLite/Writers/ContentLayoutSQLWriter.cs +++ b/Analyzer/SQLite/Writers/ContentLayoutSQLWriter.cs @@ -1,8 +1,9 @@ using System; using System.Collections.Generic; -using System.IO; +using System.Linq; using Microsoft.Data.Sqlite; using UnityDataTools.Analyzer.SQLite.Commands.ContentLayout; +using UnityDataTools.Analyzer.Util; using UnityDataTools.Models; namespace UnityDataTools.Analyzer.SQLite.Writers @@ -28,12 +29,21 @@ internal class ContentLayoutSQLWriter : IDisposable private bool m_Initialized; private SqliteConnection m_Database; + // Shared with the serialized-file analysis (see AnalyzerTool): file ids must agree + // between the layout and the analyzed content, and the dependency map is what resolves + // the external references of the analyzed .cf files. + private IdProvider m_SerializedFileIdProvider; + private ContentFileDependencyMap m_ContentFileDependencies; + // The layout files that can have a .cf file on disk, kept for LinkSerializedFiles. private List<(int Index, string ContentHash)> m_ImportedFiles = new(); - public ContentLayoutSQLWriter(SqliteConnection database) + public ContentLayoutSQLWriter(SqliteConnection database, + IdProvider serializedFileIdProvider, ContentFileDependencyMap contentFileDependencies) { m_Database = database; + m_SerializedFileIdProvider = serializedFileIdProvider; + m_ContentFileDependencies = contentFileDependencies; } // Creates the content_layout tables and views. Called lazily on the first import so that @@ -174,23 +184,50 @@ public void WriteContentLayout(string filename, ContentLayout layout) // Indexes are created after the bulk insert so that a very large layout imports fast. ExecuteDDL(Properties.Resources.ContentLayoutIndexes); + + PopulateContentFileDependencies(layout); } - // Fills in the serialized_file column, linking each layout entry to the serialized_files - // row of its analyzed .cf file. Called after all files are processed; entries whose file - // was not part of the analyzed input (e.g. a layout-only analyze) stay NULL. The match is - // on the file name (the content hash), ignoring any directory part that the analyze pass - // recorded in serialized_files.name. + // Fills the shared dependency map that resolves the external references of the analyzed + // .cf files (see ContentFileDependencyMap): for each content file, the dependency indexes + // become the target filenames, in order. Built-in dependencies (no content hash) map to + // null so the reference falls back to the external table path. + private void PopulateContentFileDependencies(ContentLayout layout) + { + var hashByIndex = (layout.SerializedFiles ?? []).ToDictionary(f => f.Index, f => f.ContentHash); + + foreach (var file in layout.SerializedFiles ?? []) + { + if (string.IsNullOrEmpty(file.ContentHash)) + continue; + + var resolved = (file.SerializedFileDependencies ?? []) + .Select(i => hashByIndex.TryGetValue(i, out var hash) && !string.IsNullOrEmpty(hash) + ? (hash + ".cf").ToLowerInvariant() + : null) + .ToArray(); + + m_ContentFileDependencies.Add((file.ContentHash + ".cf").ToLowerInvariant(), resolved); + } + } + + // Fills in the serialized_file column, linking each layout entry to its serialized_files + // row. Called after all files are processed. Ids come from the shared IdProvider, so + // entries whose .cf file was analyzed link to the row the analyze pass wrote (loose or + // inside an archive, with its archive column intact). For files never encountered on the + // input (a layout-only analyze, or a subset of a build) a placeholder serialized_files + // row is written - name only, archive NULL, no objects - so the link is always valid, + // mirroring what the dangling-refs finalize does for referenced-but-unanalyzed files. public void LinkSerializedFiles() { - var fileNameToId = new Dictionary(); + var existingIds = new HashSet(); using (var select = m_Database.CreateCommand()) { - select.CommandText = "SELECT id, name FROM serialized_files"; + select.CommandText = "SELECT id FROM serialized_files"; using var reader = select.ExecuteReader(); while (reader.Read()) { - fileNameToId[Path.GetFileName(reader.GetString(1)).ToLowerInvariant()] = reader.GetInt32(0); + existingIds.Add(reader.GetInt32(0)); } } @@ -201,15 +238,27 @@ public void LinkSerializedFiles() update.Parameters.Add("@id", SqliteType.Integer); update.Parameters.Add("@file_index", SqliteType.Integer); + var addStubRow = new Commands.SerializedFile.AddSerializedFile(); + addStubRow.CreateCommand(m_Database); + addStubRow.SetTransaction(transaction); + try { foreach (var file in m_ImportedFiles) { - if (fileNameToId.TryGetValue(file.ContentHash + ".cf", out var id)) + var fileName = (file.ContentHash + ".cf").ToLowerInvariant(); + var id = m_SerializedFileIdProvider.GetId(fileName); + + update.Parameters["@id"].Value = id; + update.Parameters["@file_index"].Value = file.Index; + update.ExecuteNonQuery(); + + if (existingIds.Add(id)) { - update.Parameters["@id"].Value = id; - update.Parameters["@file_index"].Value = file.Index; - update.ExecuteNonQuery(); + addStubRow.SetValue("id", id); + addStubRow.SetValue("archive", null); + addStubRow.SetValue("name", fileName); + addStubRow.ExecuteNonQuery(); } } @@ -220,6 +269,10 @@ public void LinkSerializedFiles() transaction.Rollback(); throw; } + finally + { + addStubRow.Dispose(); + } } private void SetTransaction(SqliteTransaction transaction) diff --git a/Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs b/Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs index e5bb82d..1eaea3e 100644 --- a/Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs +++ b/Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs @@ -24,11 +24,18 @@ public class SerializedFileSQLiteWriter : IDisposable // Global id assignment shared across every serialized file in the database. // m_SerializedFileIdProvider maps a serialized file (by lowercased file name) to its - // serialized_files row id; m_ObjectIdProvider maps a (serialized file id, pathId) pair to - // its objects row id. See ObjectIdProvider for how cross-file references are resolved. - private IdProvider m_SerializedFileIdProvider = new(); + // serialized_files row id; it is owned by AnalyzerTool because the ContentLayout import + // assigns ids through the same provider. m_ObjectIdProvider maps a (serialized file id, + // pathId) pair to its objects row id. See ObjectIdProvider for how cross-file references + // are resolved. + private IdProvider m_SerializedFileIdProvider; private ObjectIdProvider m_ObjectIdProvider = new(); + // File-to-file dependencies from an imported ContentLayout.json, used to resolve the + // external references of ContentDirectory files (whose external reference table holds + // symbolic placeholders). Empty when no layout is imported. + private ContentFileDependencyMap m_ContentFileDependencies; + // The refs table stores ids into these deduplicated string tables instead of repeating the // property path/type strings on every row. Ids are assigned lazily and are global across all // files; the HashSets track which ids have already had their lookup row written. @@ -80,12 +87,15 @@ public class SerializedFileSQLiteWriter : IDisposable private SqliteConnection m_Database; private SqliteCommand m_LastId = new SqliteCommand(); private SqliteTransaction m_CurrentTransaction = null; - public SerializedFileSQLiteWriter(SqliteConnection database, bool skipReferences, bool skipCrc) + public SerializedFileSQLiteWriter(SqliteConnection database, bool skipReferences, bool skipCrc, + IdProvider serializedFileIdProvider, ContentFileDependencyMap contentFileDependencies) { m_Initialized = false; m_Database = database; m_SkipReferences = skipReferences; m_SkipCrc = skipCrc; + m_SerializedFileIdProvider = serializedFileIdProvider; + m_ContentFileDependencies = contentFileDependencies; } public void Init() @@ -236,13 +246,28 @@ public void WriteSerializedFile(string relativePath, string fullPath, string con // Local file id 0 is always this file itself; ids 1..N follow the order of the // external reference table. Resolve each external reference to its global file id - // by (lowercased) file name. + // by (lowercased) file name. For ContentDirectory files the external table holds + // symbolic placeholders, so when an imported ContentLayout covers this file the + // actual target comes from its dependency list, matched by position; built-in + // dependencies (null entries) keep the external table path. + var resolvedDependencies = m_ContentFileDependencies.GetDependencies( + Path.GetFileName(fullPath).ToLowerInvariant()); + + if (resolvedDependencies != null && resolvedDependencies.Length != sf.ExternalReferences.Count) + { + Console.Error.WriteLine( + $"Warning: {relativePath}: the external reference count ({sf.ExternalReferences.Count}) does not match " + + $"the ContentLayout dependency count ({resolvedDependencies.Length}); using the external reference table."); + resolvedDependencies = null; + } + int localId = 0; m_LocalToDbFileId.Add(localId++, serializedFileId); foreach (var extRef in sf.ExternalReferences) { - m_LocalToDbFileId.Add(localId++, - m_SerializedFileIdProvider.GetId(extRef.Path.Substring(extRef.Path.LastIndexOf('/') + 1).ToLowerInvariant())); + var name = resolvedDependencies?[localId - 1] + ?? extRef.Path.Substring(extRef.Path.LastIndexOf('/') + 1).ToLowerInvariant(); + m_LocalToDbFileId.Add(localId++, m_SerializedFileIdProvider.GetId(name)); } foreach (var obj in sf.Objects) diff --git a/Analyzer/Util/ContentFileDependencyMap.cs b/Analyzer/Util/ContentFileDependencyMap.cs new file mode 100644 index 0000000..047247e --- /dev/null +++ b/Analyzer/Util/ContentFileDependencyMap.cs @@ -0,0 +1,28 @@ +using System.Collections.Generic; + +namespace UnityDataTools.Analyzer.Util; + +// The file-to-file dependencies of a ContentDirectory build, populated from ContentLayout.json. +// In ContentDirectory builds the external reference table inside a SerializedFile holds symbolic +// .cfid placeholders instead of real filenames; the actual target of a reference is determined +// positionally through the layout's dependency list (see Documentation/contentdirectory-format.md). +// This map holds, for each content file (keyed by its lowercased ".cf" filename), +// the resolved dependency filenames in external-reference-table order. An entry is null where the +// dependency is a built-in file (which has no content hash); such references fall back to the +// path from the external reference table. +public class ContentFileDependencyMap +{ + private Dictionary m_Dependencies = new(); + + public void Add(string fileName, string[] resolvedDependencies) + { + m_Dependencies[fileName] = resolvedDependencies; + } + + // Returns the resolved dependency filenames for the given (lowercased) filename, or null if + // the file is not covered by an imported ContentLayout. + public string[] GetDependencies(string fileName) + { + return m_Dependencies.TryGetValue(fileName, out var dependencies) ? dependencies : null; + } +} diff --git a/Documentation/analyze-examples-contentlayout.md b/Documentation/analyze-examples-contentlayout.md new file mode 100644 index 0000000..b7eb712 --- /dev/null +++ b/Documentation/analyze-examples-contentlayout.md @@ -0,0 +1,101 @@ +# Example queries for ContentDirectory builds + +Example SQL queries against a database produced by running [`analyze`](command-analyze.md) on a +ContentDirectory build together with its `ContentLayout.json`. The tables and views used here are +documented in [ContentLayout in the Analyze Database](contentlayout-database.md); for general notes +on running queries (sqlite3 command line, DB Browser), see [Example usage of Analyze](analyze-examples.md). + +Some of these queries need only the layout (they work in a layout-only database); others combine +the layout with the analyzed build content and need both on the analyze input. + +## Which files was a source asset built into? (layout only) + +The same source asset can contribute to more than one file: + +```sql +SELECT * FROM content_layout_source_assets_view +WHERE asset_path = 'Assets/Textures/GreenStatic.png'; +``` + +## Build size by artifact category (layout only) + +```sql +SELECT category, COUNT(*) AS count, SUM(size) AS bytes +FROM content_layout_binary_artifacts +GROUP BY category ORDER BY bytes DESC; +``` + +## Everything required to load a source asset (layout only) + +The recursive closure over the file dependencies: starting from the file(s) containing a source +asset, every serialized file that must be loaded with them. + +```sql +WITH RECURSIVE closure(file_index) AS ( + SELECT serialized_file_index FROM content_layout_source_assets + WHERE asset_path = 'Assets/ScriptableObjects/ContentDirectoryRoot.asset' + UNION + SELECT d.dependency_index + FROM content_layout_serialized_file_dependencies d + INNER JOIN closure c ON c.file_index = d.serialized_file_index +) +SELECT v.* FROM closure INNER JOIN content_layout_serialized_files_view v USING (file_index); +``` + +Add up the `size` column of the result for the total load footprint (excluding data loaded on +demand through loadables, and the `.resS`/`.resource` data files — join +`content_layout_data_files_view` to include those). + +## All objects built from a source asset (layout + content) + +Non-recursive: the objects inside the file(s) a given source asset was built into. For an asset +that produces many objects (an FBX, a scene) this shows what it expanded to. + +```sql +SELECT o.* +FROM content_layout_source_assets s +INNER JOIN content_layout_serialized_files f ON f.file_index = s.serialized_file_index +INNER JOIN objects o ON o.serialized_file = f.serialized_file +WHERE s.asset_path = 'Assets/Scenes/Scene1.unity'; +``` + +## The loadables and what they resolve to (layout + content) + +Each loadable with the analyzed object it points at (type, name, size), and whether it is a root +asset of the build: + +```sql +SELECT * FROM content_layout_loadable_objects_view ORDER BY is_root_asset DESC, asset_path; +``` + +## Data files of a serialized file (layout only) + +The `.resS`/`.resource` files holding the streamed texture/mesh and audio/video data used by each +serialized file: + +```sql +SELECT * FROM content_layout_data_files_view +WHERE filename = 'c0152db4dd710be51b2decb997325f34.cf'; +``` + +## Checking reference resolution (layout + content) + +When the layout was part of the analyze, the references between Content Files are resolved, so the +only expected entries in [`dangling_refs`](analyzer.md#dangling_refs--dangling_refs_view) are the +references into Unity's built-in resources: + +```sql +SELECT DISTINCT sf.name FROM dangling_refs d +INNER JOIN serialized_files sf ON sf.id = d.serialized_file; +-- expected: 'unity default resources' only +``` + +Anything else listed here means part of the build was missing from the analyzed input. + +## Related documentation + +| Topic | Description | +|-------|-------------| +| [ContentLayout in the Analyze Database](contentlayout-database.md) | The tables and views these queries use. | +| [Example usage of Analyze](analyze-examples.md) | General analyze query examples and command-line usage. | +| [Content Directory Format](contentdirectory-format.md) | The build output format and its reference conventions. | diff --git a/Documentation/analyze-examples.md b/Documentation/analyze-examples.md index 439baf5..428cd30 100644 --- a/Documentation/analyze-examples.md +++ b/Documentation/analyze-examples.md @@ -6,6 +6,8 @@ The command line arguments to invoke Analyze are documented [here](unitydatatool The definition of the views, and some internal details about how Analyze is implemented, can be found [here](analyzer.md). +Queries specific to ContentDirectory builds and `ContentLayout.json` are collected on a dedicated page: [Example queries for ContentDirectory builds](analyze-examples-contentlayout.md). + ## Running Queries from the Command line You can find data in the SQLite database by running SQL queries. diff --git a/Documentation/command-analyze.md b/Documentation/command-analyze.md index 14faba6..6cbfbf7 100644 --- a/Documentation/command-analyze.md +++ b/Documentation/command-analyze.md @@ -67,6 +67,8 @@ command works with the following types of input: | **Entities content** | `StreamingAssets/ContentArchives` folder for [Entities](https://docs.unity3d.com/Packages/com.unity.entities@1.4/manual/content-management-intro.html) projects | | **Player Data folder** | The `Data` folder of a Unity Player build | | **Compressed Player builds** | The `data.unity3d` file will be analyzed like AssetBundles | +| **ContentDirectory build output** | The output of [`BuildPipeline.BuildContentDirectory`](https://docs.unity3d.com/6000.6/Documentation/ScriptReference/BuildPipeline.BuildContentDirectory.html) (Unity 6.6+), ideally together with its build history folder. See [ContentDirectory builds](#contentdirectory-builds) | +| **ContentLayout.json** | The layout file of a ContentDirectory build, imported into dedicated tables. Also useful on its own for querying a large layout with SQL. See [ContentLayout in the Analyze Database](contentlayout-database.md) | | **BuildReport files** | The build report is typically found at a path like `Library/LastBuild.buildreport`and is a binary serialized file | | **AssetDatabase Artifacts** | The tool will work to some extent with serialized files created in the AssetDatabase artifact storage, inside the Library folder | @@ -74,6 +76,42 @@ command works with the following types of input: --- +## ContentDirectory builds + +Analyze a ContentDirectory build together with its `ContentLayout.json` by passing the build output +folder and the build report folder (or the layout file itself) in one call: + +```bash +UnityDataTool analyze /path/to/ContentDirectory /path/to/Library/BuildHistory/ -o database.db +``` + +The layout provides the source-asset mapping (imported as the `content_layout` tables, see +[ContentLayout in the Analyze Database](contentlayout-database.md)) and the dependency information +that resolves the references between the build's Content Files (see +[Content Directory Format](contentdirectory-format.md)). Without it, cross-file references cannot be +resolved and are recorded in `dangling_refs`. + +A `ContentLayout.json` is only used when its `BuildManifestHash` matches the analyzed build's +`BuildManifestHash.txt` (a file guaranteed to exist in the build output folder, which analyze picks +up automatically). This exact-match rule prevents a stale or unrelated layout from silently +producing misleading results, and it means you can point analyze at a folder containing several +layouts (such as `Library/BuildHistory`) and the matching one is selected. + +How the input combinations behave: + +| Input | Result | +|-------|--------| +| ContentDirectory only | Analyzed, with a warning that the analysis is incomplete (broken references, no source mapping) | +| Subset of files from a ContentDirectory | Same as above; recognized by the `.cf` extension | +| ContentDirectory + matching ContentLayout.json | Analyzed with resolved references and source mapping | +| Subset of ContentDirectory files + ContentLayout.json | Works when a `BuildManifestHash.txt` can be found for the hash match; error when the layout cannot be validated | +| ContentDirectory + multiple ContentLayout.json | The one matching the build is used, the rest are ignored; error when none match | +| ContentLayout.json only | Imported on its own (layout tables only) | +| Multiple ContentDirectory builds | Error — analyze a single build at a time | +| Non-ContentDirectory content + ContentDirectory | Allowed, e.g. a Player build and a ContentDirectory together for duplicate detection | + +--- + ## Output Database The analysis creates a SQLite database that can be explored using tools like [DB Browser for SQLite](https://sqlitebrowser.org/) or the command line `sqlite3` tool. diff --git a/Documentation/command-find-refs.md b/Documentation/command-find-refs.md index e2e26fe..b492737 100644 --- a/Documentation/command-find-refs.md +++ b/Documentation/command-find-refs.md @@ -28,6 +28,8 @@ UnityDataTool find-refs [options] This command requires a database created by the [`analyze`](command-analyze.md) command **without** the `--skip-references` option. +Reference chains end at an asset: an AssetBundle's assets, or — for ContentDirectory builds — the loadable objects described by the build's `ContentLayout.json`. For a ContentDirectory build, include the `ContentLayout.json` in the analyze input, both to resolve the references between content files and to provide those chain roots (see [ContentLayout in the Analyze Database](contentlayout-database.md)). + --- ## Examples diff --git a/Documentation/contentdirectory-format.md b/Documentation/contentdirectory-format.md index 00f10d0..f4192fa 100644 --- a/Documentation/contentdirectory-format.md +++ b/Documentation/contentdirectory-format.md @@ -211,8 +211,9 @@ Two files in the build history are directly relevant here: * **[`ContentLayout.json`](contentlayout.md)** — maps the built content back to the source assets in the project and describes the dependencies between the produced files. It is the key file for - understanding a content directory build. UnityDataTool support for `ContentLayout.json` is a work in - progress; for now, see the dedicated [ContentLayout.json](contentlayout.md) page for its structure. + understanding a content directory build. [`analyze`](command-analyze.md) imports it into queryable + database tables and uses it to resolve the references between Content Files (see + [ContentLayout in the Analyze Database](contentlayout-database.md)). * **The BuildReport file** — the build report for the build, in the same SerializedFile format that UnityDataTool reads for Player and AssetBundle builds. In the build history it is named after the build session GUID (rather than the fixed `LastBuild.buildreport` name), so reports from multiple @@ -312,15 +313,20 @@ paths: UnityDataTool analyze /path/to/ContentDirectory /path/to/Library/BuildHistory/ ``` -Analyzing the build output alone records the objects, but not where they came from. The build report -in the build history adds the source-asset mapping (the PackedAssets data), so analyzing the two -together gives a database that ties each built object back to its source asset. See -[BuildReport Support](buildreport.md) for how the build report data is stored and queried. - -Currently the references between objects are not recorded properly for a content directory build, -because those references live in the manifest rather than in the Content Files themselves (see -[References between Content Files](#references-between-content-files) above). This will be resolved -using the information from the `ContentLayout.json` file. +Analyzing the build output alone records the objects, but not where they came from — and because the +references between Content Files live in the manifest rather than in the files themselves (see +[References between Content Files](#references-between-content-files) above), those references end up +in the `dangling_refs` table and analyze prints a warning. Including the build history folder fixes +both: the `ContentLayout.json` it contains provides the source-asset mapping (imported as the +`content_layout` tables, see [ContentLayout in the Analyze Database](contentlayout-database.md)) and +the dependency information that analyze uses to resolve the references between Content Files. The +build report adds the per-object source mapping (the PackedAssets data, see +[BuildReport Support](buildreport.md)). + +Analyze verifies that the `ContentLayout.json` matches the build through the build's +`BuildManifestHash.txt`, so a stale layout from a different build is rejected rather than producing +misleading results. See the [`analyze` command](command-analyze.md#contentdirectory-builds) page for +the exact input combinations. ## Related documentation @@ -329,6 +335,7 @@ using the information from the `ContentLayout.json` file. | [Use content directories to load assets at runtime](https://docs.unity3d.com/6000.6/Documentation/Manual/content-directories.html) | Unity Manual: what content directories are, and the APIs to build and load them. | | [Analyze builds](https://docs.unity3d.com/6000.6/Documentation/Manual/build-analyze-builds.html) | Unity Manual: the build history and the build report. | | [ContentLayout.json](contentlayout.md) | The build layout file that maps content directory output back to source assets. | +| [ContentLayout in the Analyze Database](contentlayout-database.md) | How `analyze` imports `ContentLayout.json` into queryable database tables. | | [BuildReport Support](buildreport.md) | Analyzing Unity build report files with UnityDataTool. | | [Overview of Unity Content](unity-content-format.md) | SerializedFiles, Unity Archives, and TypeTrees. | | [AssetBundle Format](assetbundle-format.md) | The earlier system that content directories are a newer alternative to. | diff --git a/Documentation/contentlayout-database.md b/Documentation/contentlayout-database.md index 29f8c87..606b9fe 100644 --- a/Documentation/contentlayout-database.md +++ b/Documentation/contentlayout-database.md @@ -2,12 +2,14 @@ When the input of the [`analyze`](command-analyze.md) command includes the `ContentLayout.json` of a content directory build, its content is imported into the database tables and views documented on this page. This makes the layout of the build queryable with SQL — which source assets each file was built from, the dependencies between the files, the loadable objects and scenes, and the size of every artifact — and connects that information to the objects and references that `analyze` extracts from the build content itself. -For what `ContentLayout.json` contains conceptually, see [ContentLayout.json](contentlayout.md); for the build output format it describes, see [Content Directory Format](contentdirectory-format.md). +The layout is also what makes reference information correct for a ContentDirectory build: the external reference tables inside the build's files hold symbolic placeholders, and analyze resolves them through the layout's dependency lists. With the layout on the input, references between content files land in the regular `refs` table (and [`find-refs`](command-find-refs.md) works across files); without it they are recorded in [`dangling_refs`](analyzer.md#dangling_refs--dangling_refs_view). + +For what `ContentLayout.json` contains conceptually, see [ContentLayout.json](contentlayout.md); for the build output format it describes and the reference-resolution mechanics, see [Content Directory Format](contentdirectory-format.md). For the input combinations analyze accepts (including how the layout is matched to the build), see [the `analyze` command](command-analyze.md#contentdirectory-builds). Example queries: [Example queries for ContentDirectory builds](analyze-examples-contentlayout.md). ## General notes * The `content_layout*` tables and views are only created when a `ContentLayout.json` is actually part of the analyzed input, so databases produced from AssetBundles or Player builds are not cluttered with empty tables. -* A single layout per database is supported. Additional `ContentLayout.json` files on the input are reported as failed. +* A single layout per database is supported. When several `ContentLayout.json` files are on the input, the one whose `BuildManifestHash` matches the analyzed build is selected and the rest are ignored (see [the `analyze` command](command-analyze.md#contentdirectory-builds)). * The tables mirror the json structure. Entries that cross-reference each other by array index in the json do the same in the database: `file_index` and `artifact_index` columns hold the json array indexes. * Two adjustments are made so the data is natural to query: * The json's sentinel values are stored as SQL `NULL`: a `SerializedFile` value of `-1` (an object dropped from the build) and the missing `ContentHash` of built-in entries. @@ -24,7 +26,7 @@ One row identifying the imported layout: `name` (the path of the imported file), One row per serialized file (`.cf` Content File) of the build, keyed by `file_index`. Records the symbolic `cfid` reference string, the `is_builtin` flag, and the `content_hash` that gives the filename (`content_hash || '.cf'`). -The `serialized_file` column references the core `serialized_files` table when the build content is part of the analyzed input, connecting the layout to the analyzed objects (see [below](#analyzing-the-layout-with-or-without-the-build-content)). +The `serialized_file` column references the core `serialized_files` table, connecting the layout to the analyzed objects (see [below](#analyzing-the-layout-with-or-without-the-build-content) for what the referenced row contains when the build content was not analyzed). ### content_layout_source_assets @@ -76,10 +78,10 @@ FROM content_layout_binary_artifacts GROUP BY category ORDER BY bytes DESC; ## Analyzing the layout with or without the build content -A `ContentLayout.json` can be analyzed on its own — useful for running SQL queries against a large layout — or together with the build output it describes. The layout tables themselves are identical in both cases; what differs is the connection to the core tables: +A `ContentLayout.json` can be analyzed on its own — useful for running SQL queries against a large layout — or together with the build output it describes. The layout tables themselves are identical in both cases; what differs is what the `serialized_file` link points at: -* When the build content is part of the analyzed input, each `content_layout_serialized_files` row is linked to its analyzed file through the `serialized_file` column, and the views resolve across that link (e.g. `content_layout_loadable_objects_view` shows the object, type, name and size of each loadable). -* In a layout-only analyze there is nothing to link to, so expect NULL in `content_layout_serialized_files.serialized_file`, in the `archive` column of `content_layout_serialized_files_view`, and in the `object`, `type`, `name` and `size` columns of `content_layout_loadable_objects_view`. +* When the build content is part of the analyzed input, each `content_layout_serialized_files` row links to its analyzed file, and the views resolve across that link (e.g. `content_layout_loadable_objects_view` shows the object, type, name and size of each loadable). +* When a file was not part of the analyzed input (a layout-only analyze, or a subset of a build), its layout entry links to a placeholder `serialized_files` row instead: the row holds only the filename, with `archive` NULL and no objects. The link column is therefore always valid, and whether a file was actually analyzed is visible through its objects: `EXISTS (SELECT 1 FROM objects o WHERE o.serialized_file = f.serialized_file)`. In a layout-only database, expect NULL in the `archive` column of `content_layout_serialized_files_view` and in the `object`, `type`, `name` and `size` columns of `content_layout_loadable_objects_view`. * Built-in entries (`is_builtin = 1`) always have a NULL `content_hash` and `serialized_file` — they are not files produced by the build. ## Related documentation @@ -88,5 +90,6 @@ A `ContentLayout.json` can be analyzed on its own — useful for running SQL que |-------|-------------| | [ContentLayout.json](contentlayout.md) | What the file contains and its json schema. | | [Content Directory Format](contentdirectory-format.md) | Content directory builds and inspecting them with UnityDataTool. | -| [`analyze` command](command-analyze.md) | The command that imports the layout. | +| [`analyze` command](command-analyze.md) | The command that imports the layout, and the accepted input combinations. | +| [Example queries for ContentDirectory builds](analyze-examples-contentlayout.md) | Example SQL queries against these tables. | | [Analyzer](analyzer.md) | The core database schema (objects, serialized files, references). | diff --git a/ReferenceFinder/ReferenceFinderTool.cs b/ReferenceFinder/ReferenceFinderTool.cs index 4903b62..2edc9e7 100644 --- a/ReferenceFinder/ReferenceFinderTool.cs +++ b/ReferenceFinder/ReferenceFinderTool.cs @@ -28,6 +28,7 @@ public class ReferenceFinderTool SqliteCommand m_GetObjectCommand; List m_Roots = new List(); HashSet<(long, string)> m_ProcessedObjects = new HashSet<(long, string)>(); + HashSet m_LoadableObjectIds = new HashSet(); TextWriter m_Writer; @@ -141,6 +142,8 @@ int FindReferences(SqliteConnection db, string outputFile, IList objectIds m_GetRefsCommand.CommandText = @"SELECT object, property_path, EXISTS (SELECT * FROM assetbundle_assets a WHERE a.object = r.object) FROM refs_view r WHERE referenced_object = @id"; m_GetRefsCommand.Parameters.Add("@id", SqliteType.Integer); + ReadContentLayoutLoadableObjects(db); + // Resolve the 'm_Script' property path to its id once so the per-object script lookup below // filters on the indexed integer column instead of scanning the property_names table. long scriptPathId = -1; @@ -193,17 +196,38 @@ FROM object_view o command.CommandText = "SELECT asset_name, archive, serialized_file FROM assetbundle_asset_view WHERE id = @id"; + // Chain roots from a ContentDirectory build are loadable objects, described by the + // imported ContentLayout rather than by an AssetBundle's asset table. + using var loadableCommand = m_LoadableObjectIds.Count > 0 ? db.CreateCommand() : null; + if (loadableCommand != null) + { + loadableCommand.CommandText = "SELECT asset_path, filename FROM content_layout_loadable_objects_view WHERE object = @id"; + loadableCommand.Parameters.Add("@id", SqliteType.Integer); + } + foreach (var root in m_Roots) { command.Parameters["@id"].Value = root.Id; using (var reader = command.ExecuteReader()) { - reader.Read(); - - m_Writer.WriteLine("Found reference in:"); - m_Writer.WriteLine(reader.GetString(0)); - m_Writer.WriteLine($"(Archive = {reader.GetString(1)}; SerializedFile = {reader.GetString(2)})"); + if (reader.Read()) + { + m_Writer.WriteLine("Found reference in:"); + m_Writer.WriteLine(reader.GetString(0)); + m_Writer.WriteLine($"(Archive = {reader.GetString(1)}; SerializedFile = {reader.GetString(2)})"); + } + else if (loadableCommand != null) + { + loadableCommand.Parameters["@id"].Value = root.Id; + using var loadableReader = loadableCommand.ExecuteReader(); + if (loadableReader.Read()) + { + m_Writer.WriteLine("Found reference in loadable:"); + m_Writer.WriteLine(loadableReader.GetString(0)); + m_Writer.WriteLine($"(SerializedFile = {loadableReader.GetString(1)})"); + } + } } OutputReferenceNode(root, "", 1); @@ -270,6 +294,28 @@ void OutputReferenceNode(ReferenceTreeNode node, string propertyPath, int indent } } + // Loads the analyzed object ids of the loadables described by an imported ContentLayout. + // These are the addressable entry points of a ContentDirectory build, so like an + // AssetBundle's assets they complete a reference chain. Empty when the database has no + // content_layout tables (they are only created when a ContentLayout.json was analyzed). + void ReadContentLayoutLoadableObjects(SqliteConnection db) + { + using var checkCmd = db.CreateCommand(); + checkCmd.CommandText = "SELECT EXISTS (SELECT 1 FROM sqlite_master WHERE type = 'view' AND name = 'content_layout_loadable_objects_view')"; + if ((long)checkCmd.ExecuteScalar() == 0) + { + return; + } + + using var loadablesCmd = db.CreateCommand(); + loadablesCmd.CommandText = "SELECT object FROM content_layout_loadable_objects_view WHERE object IS NOT NULL"; + using var reader = loadablesCmd.ExecuteReader(); + while (reader.Read()) + { + m_LoadableObjectIds.Add(reader.GetInt64(0)); + } + } + ReferenceTreeNode ProcessReferences(long id, bool findAll) { var references = new List<(long id, string propertyPath, bool isAsset)>(); @@ -280,7 +326,9 @@ ReferenceTreeNode ProcessReferences(long id, bool findAll) { while (reader.Read()) { - references.Add((reader.GetInt64(0), reader.GetString(1), reader.GetBoolean(2))); + var referencingObject = reader.GetInt64(0); + references.Add((referencingObject, reader.GetString(1), + reader.GetBoolean(2) || m_LoadableObjectIds.Contains(referencingObject))); } } diff --git a/UnityDataTool.Tests/AnalyzeContentLayoutTests.cs b/UnityDataTool.Tests/AnalyzeContentLayoutTests.cs index 97c4429..912f444 100644 --- a/UnityDataTool.Tests/AnalyzeContentLayoutTests.cs +++ b/UnityDataTool.Tests/AnalyzeContentLayoutTests.cs @@ -1,3 +1,4 @@ +using System; using System.IO; using System.Linq; using System.Threading.Tasks; @@ -107,10 +108,21 @@ SELECT dependency_index FROM content_layout_serialized_file_dependencies "Assets/ScriptableObjects/ContentDirectoryRoot.asset", "ContentDirectoryRoot is the only root asset"); - // The core-table link is only populated when the build content is analyzed too. + // Even without the build content, every non-built-in entry links to a serialized_files + // row: a placeholder holding just the filename (archive NULL, no objects). SQLTestHelper.AssertQueryInt(db, - "SELECT COUNT(*) FROM content_layout_serialized_files WHERE serialized_file IS NOT NULL", 0, - "a layout-only analyze cannot link to analyzed serialized files"); + "SELECT COUNT(*) FROM content_layout_serialized_files WHERE is_builtin = 0 AND serialized_file IS NULL", 0, + "a layout-only analyze links every entry to a placeholder row"); + SQLTestHelper.AssertQueryInt(db, + @"SELECT COUNT(*) FROM content_layout_serialized_files f + INNER JOIN serialized_files sf ON sf.id = f.serialized_file + WHERE sf.name != f.content_hash || '.cf' OR sf.archive IS NOT NULL", 0, + "placeholder rows hold the filename and no archive"); + SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM objects", 0, + "a layout-only analyze produces no objects"); + SQLTestHelper.AssertQueryInt(db, + "SELECT COUNT(*) FROM content_layout_loadable_objects_view WHERE object IS NOT NULL", 0, + "loadables cannot resolve to objects in a layout-only analyze"); SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM content_layout_binary_artifacts WHERE category = 'contentfile'", 13, "one contentfile artifact per non-built-in serialized file"); @@ -180,6 +192,215 @@ AND NOT EXISTS (SELECT 1 FROM objects o WHERE o.serialized_file = f.serialized_f @"SELECT type FROM content_layout_loadable_objects_view WHERE asset_path = 'Assets/ScriptableObjects/ContentDirectoryRoot.asset'", "MonoBehaviour", "ScriptableObjects are serialized as MonoBehaviour"); + + // The layout resolves the .cfid placeholder references, so the only dangling targets + // left are Unity's built-in resources (shipped without TypeTrees, never analyzed). + SQLTestHelper.AssertQueryString(db, + @"SELECT GROUP_CONCAT(DISTINCT sf.name) FROM dangling_refs d + INNER JOIN serialized_files sf ON sf.id = d.serialized_file", + "unity default resources", "only built-in references should dangle"); + Assert.Greater(SQLTestHelper.QueryInt(db, + @"SELECT COUNT(*) FROM refs r + INNER JOIN objects a ON a.id = r.object + INNER JOIN objects b ON b.id = r.referenced_object + WHERE a.serialized_file != b.serialized_file"), 0, + "references between content files should resolve to analyzed objects"); + + // The known chain of the reference build: the ContentDirectoryRoot ScriptableObject + // directly references the SerializationDemo ScriptableObject in another content file. + SQLTestHelper.AssertQueryInt(db, + @"SELECT COUNT(*) FROM refs r + INNER JOIN objects src ON src.id = r.object + INNER JOIN objects tgt ON tgt.id = r.referenced_object + WHERE src.name = 'ContentDirectoryRoot' AND tgt.name = 'SerializationDemo' + AND src.serialized_file != tgt.serialized_file", 1, + "ContentDirectoryRoot should reference SerializationDemo across files"); + + // The scenes land as analyzed files, linked from the layout (issue #97 follow-up). + SQLTestHelper.AssertQueryInt(db, + @"SELECT COUNT(*) FROM content_layout_loadable_scenes s + INNER JOIN content_layout_serialized_files f ON f.file_index = s.serialized_file_index + WHERE EXISTS (SELECT 1 FROM objects o WHERE o.serialized_file = f.serialized_file)", 2, + "both loadable scenes should link to analyzed content files"); + } + + // Runs a command capturing stderr, where analyze prints its warnings. + private static async Task<(int exitCode, string stdErr)> RunCapturingStdErr(params string[] args) + { + using var sw = new StringWriter(); + var currentError = Console.Error; + int exitCode; + try + { + Console.SetError(sw); + exitCode = await Program.Main(args); + } + finally + { + Console.SetError(currentError); + } + + return (exitCode, sw.ToString()); + } + + [Test] + public async Task Analyze_ContentDirectoryWithoutLayout_WarnsAndRecordsDanglingRefs() + { + var contentDirectory = Path.Combine(TestContext.CurrentContext.TestDirectory, + "Data", "LeadingEdgeBuilds", "ContentDirectory"); + var databasePath = SQLTestHelper.GetDatabasePath(m_TestOutputFolder); + + var (exitCode, stdErr) = await RunCapturingStdErr("analyze", contentDirectory, "-o", databasePath); + + Assert.AreEqual(0, exitCode, "a ContentDirectory without its layout is still analyzable"); + Assert.That(stdErr, Does.Contain("without its ContentLayout.json"), + "the incomplete analysis should be called out"); + + using var db = SQLTestHelper.OpenDatabase(databasePath); + SQLTestHelper.AssertQueryInt(db, + "SELECT COUNT(*) FROM sqlite_master WHERE name LIKE 'content_layout%'", 0, + "no layout tables without a layout"); + Assert.Greater(SQLTestHelper.QueryInt(db, + @"SELECT COUNT(*) FROM dangling_refs d + INNER JOIN serialized_files sf ON sf.id = d.serialized_file + WHERE sf.name LIKE '%.cfid'"), 0, + "without the layout, cross-file references dangle on their .cfid placeholders"); + } + + [Test] + public async Task Analyze_MultipleContentDirectories_Fails() + { + var contentDirectory = Path.Combine(TestContext.CurrentContext.TestDirectory, + "Data", "LeadingEdgeBuilds", "ContentDirectory"); + var otherBuild = Path.Combine(TestContext.CurrentContext.TestDirectory, + "Data", "contentdirectory-zstd"); + var databasePath = SQLTestHelper.GetDatabasePath(m_TestOutputFolder); + + var (exitCode, stdErr) = await RunCapturingStdErr( + "analyze", contentDirectory, otherBuild, "-o", databasePath); + + Assert.AreEqual(1, exitCode, "analyzing two different ContentDirectory builds must fail"); + Assert.That(stdErr, Does.Contain("more than one ContentDirectory build")); + Assert.IsFalse(File.Exists(databasePath), "no partial database should be left behind"); + } + + [Test] + public async Task Analyze_ContentDirectoryWithWrongLayout_Fails() + { + // The zstd reference build is a different build, so the LeadingEdge layout cannot match. + var otherBuild = Path.Combine(TestContext.CurrentContext.TestDirectory, + "Data", "contentdirectory-zstd"); + var databasePath = SQLTestHelper.GetDatabasePath(m_TestOutputFolder); + + var (exitCode, stdErr) = await RunCapturingStdErr( + "analyze", otherBuild, m_ContentLayoutPath, "-o", databasePath); + + Assert.AreEqual(1, exitCode, "a layout that does not match the build must not be used"); + Assert.That(stdErr, Does.Contain("matches the analyzed build")); + } + + [Test] + public async Task Analyze_MultipleLayoutCandidates_SelectsTheMatchingHash() + { + // A stale layout (wrong hash) ahead of the real one on the input: the matching one is + // selected and the stale one ignored. This is the Library/BuildHistory convenience case. + var staleFolder = Path.Combine(m_TestOutputFolder, "stale_layout"); + Directory.CreateDirectory(staleFolder); + File.WriteAllText(Path.Combine(staleFolder, "ContentLayout.json"), + "{\"Version\":2,\"BuildManifestHash\":\"deadbeefdeadbeefdeadbeefdeadbeef\"}"); + var contentDirectory = Path.Combine(TestContext.CurrentContext.TestDirectory, + "Data", "LeadingEdgeBuilds", "ContentDirectory"); + var databasePath = SQLTestHelper.GetDatabasePath(m_TestOutputFolder); + + var (exitCode, stdErr) = await RunCapturingStdErr( + "analyze", staleFolder, m_ContentLayoutPath, contentDirectory, "-o", databasePath); + + Assert.AreEqual(0, exitCode); + Assert.That(stdErr, Does.Contain("Ignoring"), "the stale layout should be reported as ignored"); + + using var db = SQLTestHelper.OpenDatabase(databasePath); + SQLTestHelper.AssertQueryString(db, "SELECT build_manifest_hash FROM content_layout", + "baff06b928d147276f2245dd3b19216a", "the matching layout should be the imported one"); + } + + [Test] + public async Task Analyze_SubsetOfContentDirectoryWithLayout_ResolvesReferences() + { + // A single .cf file plus the BuildManifestHash.txt identifying its build: the layout is + // validated through the hash file and the file's references resolve to the actual .cf + // filenames of its (un-analyzed) dependencies instead of dangling on .cfid placeholders. + var contentDirectory = Path.Combine(TestContext.CurrentContext.TestDirectory, + "Data", "LeadingEdgeBuilds", "ContentDirectory"); + var subsetFolder = Path.Combine(m_TestOutputFolder, "subset"); + Directory.CreateDirectory(subsetFolder); + const string rootFile = "c0152db4dd710be51b2decb997325f34.cf"; + File.Copy(Path.Combine(contentDirectory, rootFile), Path.Combine(subsetFolder, rootFile)); + File.Copy(Path.Combine(contentDirectory, "BuildManifestHash.txt"), + Path.Combine(subsetFolder, "BuildManifestHash.txt")); + var databasePath = SQLTestHelper.GetDatabasePath(m_TestOutputFolder); + + Assert.AreEqual(0, await Program.Main(new string[] { "analyze", subsetFolder, m_ContentLayoutPath, "-o", databasePath })); + using var db = SQLTestHelper.OpenDatabase(databasePath); + + SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM content_layout", 1, + "the layout should be imported"); + SQLTestHelper.AssertQueryInt(db, + @"SELECT COUNT(*) FROM dangling_refs d + INNER JOIN serialized_files sf ON sf.id = d.serialized_file + WHERE sf.name LIKE '%.cfid'", 0, + "no reference should dangle on a .cfid placeholder"); + Assert.Greater(SQLTestHelper.QueryInt(db, + @"SELECT COUNT(*) FROM dangling_refs d + INNER JOIN serialized_files sf ON sf.id = d.serialized_file + WHERE sf.name LIKE '%.cf'"), 0, + "references into the un-analyzed dependencies resolve to their actual filenames"); + } + + [Test] + public async Task Analyze_SubsetWithLayoutButNoHashFile_Fails() + { + // Without a BuildManifestHash.txt the layout cannot be validated against the content, + // so the analyze fails rather than producing potentially misleading results. + var contentDirectory = Path.Combine(TestContext.CurrentContext.TestDirectory, + "Data", "LeadingEdgeBuilds", "ContentDirectory"); + var subsetFolder = Path.Combine(m_TestOutputFolder, "subset_no_hash"); + Directory.CreateDirectory(subsetFolder); + const string rootFile = "c0152db4dd710be51b2decb997325f34.cf"; + File.Copy(Path.Combine(contentDirectory, rootFile), Path.Combine(subsetFolder, rootFile)); + var databasePath = SQLTestHelper.GetDatabasePath(m_TestOutputFolder); + + var (exitCode, stdErr) = await RunCapturingStdErr( + "analyze", subsetFolder, m_ContentLayoutPath, "-o", databasePath); + + Assert.AreEqual(1, exitCode); + Assert.That(stdErr, Does.Contain("cannot be validated")); + } + + [Test] + public async Task FindRefs_ContentDirectoryWithLayout_WalksCrossFileChain() + { + var contentDirectory = Path.Combine(TestContext.CurrentContext.TestDirectory, + "Data", "LeadingEdgeBuilds", "ContentDirectory"); + var databasePath = SQLTestHelper.GetDatabasePath(m_TestOutputFolder); + Assert.AreEqual(0, await Program.Main(new string[] { "analyze", m_ContentLayoutPath, contentDirectory, "-o", databasePath })); + + using var sw = new StringWriter(); + var currentOut = Console.Out; + int exitCode; + try + { + Console.SetOut(sw); + exitCode = await Program.Main(new string[] + { "find-refs", databasePath, "-n", "SerializationDemo", "-t", "MonoBehaviour", "--stdout" }); + } + finally + { + Console.SetOut(currentOut); + } + + Assert.AreEqual(0, exitCode); + Assert.That(sw.ToString(), Does.Contain("ContentDirectoryRoot"), + "the chain from the root asset should be found across content files"); } [Test] @@ -231,10 +452,10 @@ public async Task Analyze_ContentLayoutWithoutContent_ImportsNothing() } [Test] - public async Task Analyze_MultipleContentLayouts_ImportsOnlyTheFirst() + public async Task Analyze_MultipleContentLayoutsWithoutBuild_Fails() { - // Two directories each containing a (identical) ContentLayout.json. Only a single layout - // per database is supported; the second one is reported as failed. + // Two ContentLayout.json files but no build content to match them against: there is no + // way to choose, and only a single layout per database is supported. var folderA = Path.Combine(m_TestOutputFolder, "layout_a"); var folderB = Path.Combine(m_TestOutputFolder, "layout_b"); Directory.CreateDirectory(folderA); @@ -243,14 +464,11 @@ public async Task Analyze_MultipleContentLayouts_ImportsOnlyTheFirst() File.Copy(m_ContentLayoutPath, Path.Combine(folderB, "ContentLayout.json")); var databasePath = SQLTestHelper.GetDatabasePath(m_TestOutputFolder); - Assert.AreEqual(0, await Program.Main(new string[] { "analyze", folderA, folderB, "-o", databasePath })); - using var db = SQLTestHelper.OpenDatabase(databasePath); + var (exitCode, stdErr) = await RunCapturingStdErr( + "analyze", folderA, folderB, "-o", databasePath); - SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM content_layout", 1, - "only a single layout should be imported"); - Assert.IsTrue(SQLTestHelper.QueryString(db, "SELECT name FROM content_layout").Contains("layout_a"), - "the first layout on the input should be the imported one"); - SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM content_layout_serialized_files", 14, - "the imported layout content should be intact"); + Assert.AreEqual(1, exitCode, "multiple layouts without build content cannot be disambiguated"); + Assert.That(stdErr, Does.Contain("multiple ContentLayout.json")); + Assert.IsFalse(File.Exists(databasePath), "no partial database should be left behind"); } }