diff --git a/Analyzer.Tests/FileDetectionTests.cs b/Analyzer.Tests/FileDetectionTests.cs index 2a48bf7..242055d 100644 --- a/Analyzer.Tests/FileDetectionTests.cs +++ b/Analyzer.Tests/FileDetectionTests.cs @@ -188,15 +188,19 @@ public void TryParseMetadata_VersionTooOld_ReturnsFalseWithMessage() [Test] public void TryParseMetadata_VersionTooNew_ReturnsFalseWithMessage() { - var headerInfo = new SerializedFileInfo { Version = 24 }; + // Far enough ahead that this does not need revisiting every time a Unity release adds a + // version. TryParseMetadata is given the header directly, so the detector's own plausibility + // range does not apply here. + var headerInfo = new SerializedFileInfo { Version = 99 }; bool result = SerializedFileDetector.TryParseMetadata("irrelevant", headerInfo, out var metadata, out var errorMessage); Assert.IsFalse(result); Assert.IsNull(metadata); Assert.IsNotNull(errorMessage); - Assert.That(errorMessage, Does.Contain("24"), "Error should mention the actual version"); - Assert.That(errorMessage, Does.Contain("23"), "Error should mention the maximum supported version"); + Assert.That(errorMessage, Does.Contain("99"), "Error should mention the actual version"); + Assert.That(errorMessage, Does.Contain(SerializedFileDetector.MaxMetadataParseVersion.ToString()), + "Error should mention the maximum supported version"); Assert.That(errorMessage, Does.Contain("UnityDataTool"), "Error should mention UnityDataTool"); } @@ -297,6 +301,63 @@ public void TryParseMetadata_PlayerNoTypeTreeLevel1_ReturnsExpectedValues() } } + [Test] + public void TryParseMetadata_V26PlayerFile_ReturnsSharedSubtreeTable() + { + var testFile = Path.Combine(m_TestDataPath, "PlayerWithTypeTreesV26", "sharedassets1.assets"); + + Assert.IsTrue(SerializedFileDetector.TryDetectSerializedFile(testFile, out var headerInfo)); + Assert.That(headerInfo.Version, Is.EqualTo(26u)); + + bool result = SerializedFileDetector.TryParseMetadata(testFile, headerInfo, out var metadata, out var errorMessage); + Assert.IsTrue(result, $"Metadata parsing should succeed. Error: {errorMessage}"); + + Assert.That(metadata.UnityVersion, Is.EqualTo("6000.7.0b2")); + Assert.That(metadata.TypeTreeCount, Is.EqualTo(2), "PreloadData and MonoBehaviour"); + Assert.That(metadata.SerializedReferenceTypeTreeCount, Is.EqualTo(1)); + + // The shared subtree table follows m_RefTypes and is what version 26 adds to the metadata. + Assert.That(metadata.SharedSubtreeCount, Is.EqualTo(7)); + Assert.That(metadata.SharedSubtrees.Length, Is.EqualTo(metadata.SharedSubtreeCount)); + + foreach (var subtree in metadata.SharedSubtrees) + { + Assert.IsTrue(subtree.Inline, "This build embeds its subtree blobs rather than extracting them"); + Assert.That(subtree.SerializedSize, Is.GreaterThan(0)); + Assert.IsFalse(subtree.ContentHash.IsZero, "A subtree is keyed by the hash of its content"); + } + } + + // The stamp at the head of a TypeTree blob tells the two version spaces apart: up to and + // including 23 it repeats the SerializedFile version, and from 24 TypeTrees are versioned + // independently starting at 32. + [TestCase("PlayerWithTypeTrees/sharedassets1.assets", 0u, TestName = "TypeTreeFormatVersion_V22_HasNoStamp")] + [TestCase("AssetBundleTypeTreeVariations/v23_Inline/prefab_with_serializedreference.serializedfile", 23u, TestName = "TypeTreeFormatVersion_V23_StampsTheFileVersion")] + [TestCase("PlayerWithTypeTreesV26/sharedassets1.assets", 33u, TestName = "TypeTreeFormatVersion_V26_StampsTheIndependentVersion")] + public void TryParseMetadata_ReportsTypeTreeFormatVersion(string relativePath, uint expected) + { + var testFile = Path.Combine(m_TestDataPath, relativePath.Replace('/', Path.DirectorySeparatorChar)); + + Assert.IsTrue(SerializedFileDetector.TryDetectSerializedFile(testFile, out var headerInfo)); + Assert.IsTrue(SerializedFileDetector.TryParseMetadata(testFile, headerInfo, out var metadata, out _)); + + foreach (var entry in metadata.TypeTrees) + Assert.That(entry.TypeTreeFormatVersion, Is.EqualTo(expected), $"persistentTypeID={entry.PersistentTypeID}"); + } + + [Test] + public void TryParseMetadata_BeforeVersion26_HasNoSharedSubtreeTable() + { + var testFile = Path.Combine(m_TestDataPath, "AssetBundleTypeTreeVariations", "v23_Inline", + "prefab_with_serializedreference.serializedfile"); + + Assert.IsTrue(SerializedFileDetector.TryDetectSerializedFile(testFile, out var headerInfo)); + Assert.IsTrue(SerializedFileDetector.TryParseMetadata(testFile, headerInfo, out var metadata, out _)); + + Assert.That(metadata.SharedSubtreeCount, Is.EqualTo(0)); + Assert.That(metadata.SharedSubtrees, Is.Empty); + } + [Test] public void TryParseMetadata_V22PrefabWithSerializedReference_ReturnsExpectedTypeTreeData() { diff --git a/Analyzer/AnalyzerTool.cs b/Analyzer/AnalyzerTool.cs index 72054af..099c080 100644 --- a/Analyzer/AnalyzerTool.cs +++ b/Analyzer/AnalyzerTool.cs @@ -120,12 +120,13 @@ public int Analyze(AnalyzeOptions options) Console.Error.WriteLine($"Skipped (no TypeTrees): {relativePath}"); countNoTypeTrees++; } - catch (SerializedFileOpenException) + catch (SerializedFileOpenException e) { // Expected failure — the file content could not be parsed. // Don't print a stack trace; it adds no value for this known failure mode. EraseProgressLine(); Console.Error.WriteLine($"Failed to open: {relativePath}"); + Console.Error.WriteLine($" {e.Message}"); countFailures++; } catch (AnalyzeDuplicateException e) diff --git a/Analyzer/PPtrAndCrcProcessor.cs b/Analyzer/PPtrAndCrcProcessor.cs index ee3e1e2..beb158e 100644 --- a/Analyzer/PPtrAndCrcProcessor.cs +++ b/Analyzer/PPtrAndCrcProcessor.cs @@ -51,6 +51,7 @@ public class PPtrAndCrcProcessor : IDisposable // State for the object currently being processed, (re)initialized by each Process() call. private long m_Offset; // current read position within m_Reader + private long m_ObjectEnd; // end of the object being processed; bounds the registry frame read private long m_ObjectId; // analyzer id of the object being processed, passed to the callback private uint m_Crc32; // CRC accumulated so far for this object @@ -85,17 +86,29 @@ public void Dispose() m_resourceReaders.Clear(); } - // Walks the serialized object rooted at `node`, whose data starts at `offset` in the reader, - // emitting every PPtr through the callback. Returns a CRC32 fingerprint of the object's content - // (0 when CRC is disabled). `objectId` is the analyzer id of this object, forwarded to the callback. - public uint Process(long objectId, long offset, TypeTreeNode node) + // Walks the serialized object rooted at `node`, whose data starts at `offset` in the reader and + // is `size` bytes long, emitting every PPtr through the callback. Returns a CRC32 fingerprint of + // the object's content (0 when CRC is disabled). `objectId` is the analyzer id of this object, + // forwarded to the callback. + public uint Process(long objectId, long offset, long size, TypeTreeNode node) { m_Offset = offset; + m_ObjectEnd = offset + size; m_ObjectId = objectId; m_Crc32 = 0; foreach (var child in node.Children) { + // A version 3 registry sits here rather than in a node (see + // ProcessManagedReferenceRegistry). It is not part of the field that follows it, so its + // references get their own path root, named as the node-described versions are. + if (child.HasSerializedRefs) + { + m_StringBuilder.Clear(); + m_StringBuilder.Append("references."); + ProcessManagedReferenceFrame(); + } + m_StringBuilder.Clear(); m_StringBuilder.Append(child.Name); ProcessNode(child, false); @@ -104,6 +117,29 @@ public uint Process(long objectId, long offset, TypeTreeNode node) return m_Crc32; } + // Walks a version 3 registry: the frame's header and tables are CRC'd as the raw bytes they + // are, then each entry's data is walked through its own type tree, exactly as the node-described + // versions are. + private void ProcessManagedReferenceFrame() + { + var registry = ManagedReferenceRegistry.ReadFrame(m_Reader, m_Offset, m_ObjectEnd - m_Offset); + var frameStart = m_Offset; + + // Everything ahead of the first blob is the header and the two tables. + AppendCrc(frameStart, (int)(registry.BlobsOffset - frameStart)); + + foreach (var entry in registry.Entries) + { + if (entry.IsNull) + continue; + + m_Offset = entry.DataOffset; + ProcessRefTypeData(entry.Rid, entry.ClassName, entry.Namespace, entry.AssemblyName); + } + + m_Offset = frameStart + registry.FrameSize; + } + private void ProcessNode(TypeTreeNode node, bool isInManagedReferenceRegistry) { if (node.IsBasicType) @@ -268,10 +304,10 @@ private void ProcessArray(TypeTreeNode node, bool isManagedReferenceRegistry, bo } // A ManagedReferenceRegistry holds the [SerializeReference] instances owned by this object. - // In YAML/JSON it is the "references:" section that always appears at the end of a - // MonoBehaviour/ScriptableObject. Each instance is stored here exactly once; the fields that - // point at it (elsewhere in the object) only store its "rid", so shared instances and cycles - // collapse to the same rid. + // In YAML/JSON it is the "references:" section of a MonoBehaviour/ScriptableObject, which + // appears at the end of the object up to Unity 6.6 and ahead of the referencing fields from + // 6.7. Each instance is stored here exactly once; the fields that point at it (elsewhere in the + // object) only store its "rid", so shared instances and cycles collapse to the same rid. // // Given this C# source: // @@ -304,10 +340,21 @@ private void ProcessArray(TypeTreeNode node, bool isManagedReferenceRegistry, bo // a different TypeTree for every entry (see ProcessManagedReferenceData) - which is exactly why // finding references inside the registry is so much more involved than for the rest of the object. // - // Two on-disk versions exist: + // Three on-disk versions exist: // version 1 - entries stored back to back and terminated by a sentinel type (see // ProcessManagedReferenceData); the rid is implied by position. // version 2 - entries stored as a "RefIds" array, each element carrying its own rid. + // version 3 - from SerializedFile version 25 (Unity 6.7). No node describes it at all: the + // registry is a self-delimiting frame of raw bytes leading the C# class's own + // data, so it sits after the built-in fields (m_GameObject, m_Name, ...) and + // before the first field the script declares - which is the field flagged + // HasSerializedRefs, whether or not that field is itself a reference. It holds a + // table of type names and a table of records indexing into it; a record can be a + // null entry, which v1 and v2 could not express. Only a root object's data + // carries a frame: the same TypeTree used to lay out an instance's data inside + // the registry keeps the flag but has no frame, which is why every walker honours + // it only while iterating a root object's fields. + // ManagedReferenceRegistry reads it; see ProcessManagedReferenceFrame here. private void ProcessManagedReferenceRegistry(TypeTreeNode node) { if (node.Children.Count < 2) @@ -393,18 +440,24 @@ bool ProcessManagedReferenceData(TypeTreeNode refTypeNode, long rid) return false; } - // The data block follows the referenced type's own TypeTree, not this object's, so look it - // up by FQN and walk it (isInManagedReferenceRegistry = true so we don't re-enter the registry). + ProcessRefTypeData(rid, className, namespaceName, assemblyName); + + return true; + } + + // The data block follows the referenced type's own TypeTree, not the containing object's, so it + // is looked up by FQN and walked with isInManagedReferenceRegistry set, which keeps the walk + // from re-entering the registry. + private void ProcessRefTypeData(long rid, string className, string namespaceName, string assemblyName) + { var refTypeTypeTree = m_SerializedFile.GetRefTypeTypeTreeRoot(className, namespaceName, assemblyName); - var size = m_StringBuilder.Length; + var pathLength = m_StringBuilder.Length; m_StringBuilder.Append("rid("); m_StringBuilder.Append(rid); m_StringBuilder.Append(").data"); ProcessNode(refTypeTypeTree, true); - m_StringBuilder.Remove(size, m_StringBuilder.Length - size); - - return true; + m_StringBuilder.Remove(pathLength, m_StringBuilder.Length - pathLength); } private void ExtractPPtr(string referencedType) diff --git a/Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs b/Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs index 759701c..af63e2f 100644 --- a/Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs +++ b/Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs @@ -180,6 +180,11 @@ public void WriteSerializedFile(string relativePath, string fullPath, string con // VFS path here may be a real file or an entry inside a mounted archive. using (var detectStream = new UnityFileStream(fullPath)) { + // A version this build cannot read reaches the native loader as a generic failure, so + // say which version it is here rather than leaving the user with "may be corrupted". + if (SerializedFileDetector.IsVersionUnsupported(detectStream, out var versionError)) + throw new SerializedFileOpenException(fullPath, versionError); + if (SerializedFileDetector.IsMissingTypeTrees(detectStream)) throw new SerializedFileOpenException(fullPath, missingTypeTrees: true); } @@ -319,7 +324,7 @@ public void WriteSerializedFile(string relativePath, string fullPath, string con m_TypeSet.Add(obj.TypeId); } - var randomAccessReader = new RandomAccessReader(sf, root, reader, offset); + var randomAccessReader = new RandomAccessReader(sf, root, reader, offset, objectSize: obj.Size); string name = string.Empty; long streamDataSize = 0; @@ -356,7 +361,7 @@ public void WriteSerializedFile(string relativePath, string fullPath, string con // still resolves referenced object ids (AddReference skips the insert). if (!m_SkipReferences || !m_SkipCrc) { - crc32 = pptrReader.Process(currentObjectId, offset, root); + crc32 = pptrReader.Process(currentObjectId, offset, obj.Size, root); } // convert this to the new syntax diff --git a/Documentation/command-dump.md b/Documentation/command-dump.md index 2e9d09a..09b28c0 100644 --- a/Documentation/command-dump.md +++ b/Documentation/command-dump.md @@ -183,6 +183,81 @@ ID: -8138362113332287275 (ClassID: 135) SphereCollider z float 0 ``` +### `[SerializeReference]` fields + +A field marked `[SerializeReference]` in C# appears in the dump with the type **`managedReference`**, +and its value is not the object - it is a number called a **`rid`** (reference id). The objects +themselves are listed together in a `references` section, once each, with their concrete C# type and +their field values. + +Take this script: + +```csharp +public class Inventory : MonoBehaviour +{ + [Serializable] + public class Item + { + public string name; + public int count; + } + + [SerializeReference] public Item primary; + [SerializeReference] public Item backup; + [SerializeReference] public Item spare; +} +``` + +with `primary` and `backup` both assigned the *same* `Item`, and `spare` left null. Dumping it: + +``` +ID: 3862108129085620391 (ClassID: 114) MonoBehaviour + m_GameObject (PPtr) + m_FileID (int) 0 + m_PathID (SInt64) -5904263129458716409 + m_Enabled (UInt8) 1 + m_Script (PPtr) + m_FileID (int) 1 + m_PathID (SInt64) 1197423208934291241 + m_Name (string) + references (ManagedReferenceRegistry) + version (int) 3 + rid(-2) ReferencedObject + null + rid(3218405420927320064) ReferencedObject + type (ReferencedManagedType) + class (string) Inventory/Item + ns (string) + asm (string) Assembly-CSharp + data ReferencedObjectData + name (string) Health potion + count (int) 3 + primary (managedReference) + rid (SInt64) 3218405420927320064 + backup (managedReference) + rid (SInt64) 3218405420927320064 + spare (managedReference) + rid (SInt64) -2 +``` + +Reading it: + +* **`primary` and `backup` show the same `rid`.** They are two references to one object, so the + object is listed once and the sharing is visible - this is the whole point of + `[SerializeReference]` over a plain serialized field, which would have stored two independent + copies. +* **`spare` shows `rid (SInt64) -2`**, the marker for null, and the matching entry is listed as + `null` with no type or data. +* **`class` is the concrete runtime type**, which can be a subclass of the field's declared type - + that is what `[SerializeReference]` is for. Nested classes use a `/`, as in `Inventory/Item`. +* The entries are listed in the order the file stores them, which is not necessarily the order the + fields appear in. + +The `version` line describes how the registry is stored rather than anything about your data. It is +`3` for content built with Unity 6.7 or newer and `2` before that, and in older files the +`references` section appears *after* the fields instead of before them. The entries mean the same +thing either way. + **Refer to the [TextDumper documentation](textdumper.md) for detailed output format explanation.** --- diff --git a/Documentation/command-serialized-file.md b/Documentation/command-serialized-file.md index 73d9839..f5e6e7f 100644 --- a/Documentation/command-serialized-file.md +++ b/Documentation/command-serialized-file.md @@ -209,7 +209,10 @@ UnityDataTool serialized-file header level0 --format json Shows information from the metadata section of a SerializedFile. This includes the Unity version, target platform, TypeTree storage mode (inline, external, or absent), and counts of the type entries recorded in the file. The JSON output includes additional per-type details; see the notes below. -Requires SerializedFile version 19 (Unity 2019.1) or newer. Files older than version 19 are not supported by this subcommand. +Requires SerializedFile version 19 (Unity 2019.1) or newer. There is also an upper bound: a file +written by a Unity version newer than this build of UnityDataTool knows about is refused rather than +misread, with a message naming the version it found and the highest one it understands. `header` +still works on any version, so it is the way to check what you have. ### Quick Reference @@ -254,7 +257,9 @@ UnityDataTool serialized-file metadata level0 --format json "serializedReferenceTypeTreeCount": 0, "typeTrees": [ ... ], "serializedReferenceTypeTrees": [ ... ], - "scriptTypes": [ ... ] + "scriptTypes": [ ... ], + "sharedSubtreeCount": 7, + "sharedSubtrees": [ ... ] } ``` @@ -274,6 +279,8 @@ The text and JSON outputs use different field names and representations for some | *(JSON only)* | `typeTrees` | Array of per-type detail objects for the regular type entries. `null` when parsing failed or was not attempted. See **Per-Type Entry Fields** below. | | *(JSON only)* | `serializedReferenceTypeTrees` | Array of per-type detail objects for the `[SerializeReference]` type entries. Empty array for files with version < 20. See **Per-Type Entry Fields** below. | | *(JSON only)* | `scriptTypes` | Array of MonoScript references for the C# types used in this file. Each entry's index corresponds to the `scriptTypeIndex` field of a type entry in `typeTrees`. See **Script Type Entry Fields** below. | +| **Shared Subtrees** | `sharedSubtreeCount` | Number of shared sub-TypeTrees the file stores, each referenced by one or more of its types. Always `0` for files with version < 26 (Unity 6.7), and omitted from the text output when `0`. | +| *(JSON only)* | `sharedSubtrees` | Array of per-subtree detail objects. Empty array for files with version < 26. See **Shared Subtree Entry Fields** below. | ### Per-Type Entry Fields @@ -288,12 +295,23 @@ Each element of `typeTrees` and `serializedReferenceTypeTrees` in the JSON outpu | `typeTreeStructureHash` | MD4 hash of the TypeTree structure as originally written; used for compatibility checking at load time. | | `typeTreeContentHash` | XXH3 hash of the TypeTree blob. All-zeros for files with version < 23. | | `typeTreeSerializedSize` | Byte size of the TypeTree blob for this entry. `0` when `inlineTypeTree` is false. | +| `typeTreeFormatVersion` | Format version stamped into the TypeTree blob itself. `0` for files with version < 23, whose blobs carry no stamp. Up to and including 23 the stamp repeats the SerializedFile version; from version 24 (Unity 6.7) TypeTrees are versioned independently, starting at 32. | | `inlineTypeTree` | `true` when the TypeTree blob is present inline in the file's metadata. | | `className` | C# class name; non-empty only for `[SerializeReference]` entries (version ≥ 21). | | `namespaceName` | C# namespace; non-empty only for `[SerializeReference]` entries (version ≥ 21). | | `assemblyName` | Assembly name; non-empty only for `[SerializeReference]` entries (version ≥ 21). | | `typeDependencies` | Array of indices into `serializedReferenceTypeTrees` listing which `[SerializeReference]` types objects of this type may hold. Empty for `[SerializeReference]` entries or files with version < 21. | +### Shared Subtree Entry Fields + +Each element of `sharedSubtrees` in the JSON output contains: + +| JSON Field | Description | +|------------|-------------| +| `contentHash` | Hash identifying the subtree's content, which is both the key its types reference it by and the key used to fetch it from an external TypeTree store. | +| `serializedSize` | Byte size of the subtree blob. `0` when the blob was extracted to an external store. | +| `inline` | `true` when the blob is stored in this file rather than an external TypeTree store. | + ### Script Type Entry Fields Each element of `scriptTypes` in the JSON output contains: diff --git a/Documentation/unity-content-format.md b/Documentation/unity-content-format.md index 1c76918..396cb36 100644 --- a/Documentation/unity-content-format.md +++ b/Documentation/unity-content-format.md @@ -82,6 +82,32 @@ Note: the `serialized-file` and `archive` command do not require TypeTrees. >[!TIP] >The `binary2text` tool supports an optional argument `-typeinfo` to enable dumping out the TypeTrees in a SerializedFile header. That is a useful way to learn more about TypeTrees and to see exactly how Unity data is represented in the binary format. +#### What changed in Unity 6.7 + +The SerializedFile format moved from version 23 to version 26 during Unity 6.7, and two of those +changes alter how a reader has to interpret a file. UnityDataTool reads both the old and the new +shape, so this matters mainly when you compare a dump from Unity 6.6 with one from 6.7. + +**Shared sub-TypeTrees.** A compound that appears in several of a file's types - `Vector3f`, +`ColorRGBA`, `PPtr` and so on - is now stored once in a table at the end of the file's metadata, +keyed by the hash of its content, and each type that uses it carries a single node referencing that +entry. The serialized objects are unchanged; only the TypeTrees are smaller. `serialized-file +metadata` reports the table as `Shared Subtrees`. + +**The `[SerializeReference]` registry.** Up to Unity 6.6 the registry was described by TypeTree +nodes like any other field, and appeared at the end of the object. From 6.7 it is a self-contained +frame that leads the C# class's own data - after the built-in fields such as `m_GameObject` and +`m_Name`, and before the first field declared by the script - and no TypeTree node describes it. +Putting the registry first is what lets a reader assign each reference as it reads the field, +instead of patching it afterwards. `dump` prints its contents in the same shape either way, so the visible difference is +the reported registry `version` (2 before, 3 from 6.7) and where it appears in the output. The new +format can also express a null reference, which the older ones could not. + +TypeTrees are also versioned independently of the SerializedFile from 6.7 onwards, starting at +version 32, so a future TypeTree change no longer bumps the file version. `serialized-file metadata +-f Json` reports each type's stamp as `typeTreeFormatVersion`: files before version 23 have no +stamp and report 0, version 23 files report 23, and 6.7 files report 32 or higher. + #### Extracted Typetrees Starting with Unity 6.5 and Addressables 2.9 it is possible to extract the TypeTrees from all the SerializedFiles in an Addressable build into a shared file. This can reduce the size of the build output, because the TypeTree information is no longer duplicated in each file. diff --git a/SerializedFile/SerializedFileTool.cs b/SerializedFile/SerializedFileTool.cs index f2ea459..a2dd997 100644 --- a/SerializedFile/SerializedFileTool.cs +++ b/SerializedFile/SerializedFileTool.cs @@ -264,6 +264,11 @@ private static void OutputMetadataText(SerializedFileMetadata metadata) Console.WriteLine($"{"TypeTree Definitions",-20} {typeTreeDefinitions}"); Console.WriteLine($"{"TypeTree Count",-20} {metadata.TypeTreeCount}"); Console.WriteLine($"{"RefType Count",-20} {metadata.SerializedReferenceTypeTreeCount}"); + + // Only files from Unity 6.7 and later have a shared subtree table, so stay quiet about it + // for the older files that make up most of what this command is pointed at. + if (metadata.SharedSubtreeCount > 0) + Console.WriteLine($"{"Shared Subtrees",-20} {metadata.SharedSubtreeCount}"); } private static void OutputMetadataJson(SerializedFileMetadata metadata) @@ -278,6 +283,13 @@ private static void OutputMetadataJson(SerializedFileMetadata metadata) typeTrees = metadata.TypeTrees?.Select(TypeTreeInfoToJson).ToArray(), serializedReferenceTypeTrees = metadata.SerializedReferenceTypeTrees?.Select(TypeTreeInfoToJson).ToArray(), scriptTypes = metadata.ScriptTypes?.Select(s => new { fileID = s.FileID, pathID = s.PathID }).ToArray(), + sharedSubtreeCount = metadata.SharedSubtreeCount, + sharedSubtrees = metadata.SharedSubtrees?.Select(t => new + { + contentHash = t.ContentHash.ToString(), + serializedSize = t.SerializedSize, + inline = t.Inline, + }).ToArray(), }; var json = JsonSerializer.Serialize(jsonObject, new JsonSerializerOptions { WriteIndented = true }); @@ -295,6 +307,7 @@ private static object TypeTreeInfoToJson(BinaryFormat.TypeTreeInfo info) typeTreeStructureHash = info.TypeTreeStructureHash.ToString(), typeTreeContentHash = info.TypeTreeContentHash.ToString(), typeTreeSerializedSize = info.TypeTreeSerializedSize, + typeTreeFormatVersion = info.TypeTreeFormatVersion, inlineTypeTree = info.InlineTypeTree, className = info.ClassName, namespaceName = info.Namespace, diff --git a/TestCommon/Data/AssetBundleTypeTreeVariations/README.md b/TestCommon/Data/AssetBundleTypeTreeVariations/README.md index 3229409..8f65f6a 100644 --- a/TestCommon/Data/AssetBundleTypeTreeVariations/README.md +++ b/TestCommon/Data/AssetBundleTypeTreeVariations/README.md @@ -4,6 +4,7 @@ This folder contains variations of the TypeTree representations in the newest Se - **v22** is used in recent versions of Unity - **v23** is introduced in Unity 6.5 +- **v26** is introduced in Unity 6.7 ## Folder Overview @@ -14,6 +15,7 @@ This folder contains variations of the TypeTree representations in the newest Se | `v23_Inline/` | v23 | Inline | Addressables | Unity 6000.6.0a1 | | `AssetBundle-NoTypeTree/` | v22 | Disabled (`DisableWriteTypeTree`) | `BuildPipeline.BuildAssetBundles` | Unity 6000.0.65f1 | | `AssetBundle-NoTypeTreeNoVersion/` | v22 | Disabled (`DisableWriteTypeTree` + `AssetBundleStripUnityVersion`) | `BuildPipeline.BuildAssetBundles` | Unity 6000.0.65f1 | +| `v26/` | v26 | Inline, with shared subtrees | `BuildPipeline.BuildAssetBundles` | Unity 6000.7.0b2 | ## Addressable Builds (v22, v23_extracted, v23_Inline) @@ -50,6 +52,29 @@ These three folders are builds of the same tiny Addressables project. They each - Serialized file extracted from `MonoScript_monoscripts_dde848dc9848681e340a8b4fa9bd7578.bundle`. - Actual name inside AssetBundle: `CAB-d57a1d89ac0708bf030936c59479c685` +## v26 + +### managedreferences.bundle + +Unity's own fixture for the version 26 format, copied from +`Tests/Unity.PureCSharpTests/UnityFileSystemApi/data/assetbundlewithsharedsubtrees` in the Unity +source tree. It is shared with that test corpus deliberately, so the two stay comparable: its +`SerializeReferencePolymorphismExample` is the same script as the one in `UnityProjects/Baseline`, +which is what lets the version 1, 2 and 3 registry layouts be compared from one source. + +It carries the `[SerializeReference]` shapes no other fixture here has, all on +`ManagedReferenceTestBehaviour`: + +- a null reference (a registry record with no type and no data) +- two instances of one type, and one instance referred to from two fields +- a `PPtr` inside a referenced object's data +- a collection of references +- a reference nested inside another, which is what puts the records in children-first order +- a `[Serializable]` class with no fields, which is a compound of no size rather than an + unresolvable one +- a plain `int` field before and after the references, which is what catches a reader that does not + step over the registry frame: it reads those two shifted rather than failing outright + ## Built-in AssetBundle Builds (AssetBundle-NoTypeTree, AssetBundle-NoTypeTreeNoVersion) These are builds made by Unity 6000.0.65f1 of a single small ScriptableObject asset (from the BuildReportInspector package test project). The built-in AssetBundle support was used (`BuildPipeline.BuildAssetBundles`). The archive files have LZMA compression. diff --git a/TestCommon/Data/AssetBundleTypeTreeVariations/v26/managedreferences.bundle b/TestCommon/Data/AssetBundleTypeTreeVariations/v26/managedreferences.bundle new file mode 100644 index 0000000..1e199f4 Binary files /dev/null and b/TestCommon/Data/AssetBundleTypeTreeVariations/v26/managedreferences.bundle differ diff --git a/TestCommon/Data/LegacyFormats/v26format.assets b/TestCommon/Data/LegacyFormats/v26format.assets new file mode 100644 index 0000000..0e0e3c6 Binary files /dev/null and b/TestCommon/Data/LegacyFormats/v26format.assets differ diff --git a/TestCommon/Data/PlayerWithTypeTreesV26/LastBuild.buildreport b/TestCommon/Data/PlayerWithTypeTreesV26/LastBuild.buildreport new file mode 100644 index 0000000..89efb99 Binary files /dev/null and b/TestCommon/Data/PlayerWithTypeTreesV26/LastBuild.buildreport differ diff --git a/TestCommon/Data/PlayerWithTypeTreesV26/README.md b/TestCommon/Data/PlayerWithTypeTreesV26/README.md new file mode 100644 index 0000000..675e9e4 --- /dev/null +++ b/TestCommon/Data/PlayerWithTypeTreesV26/README.md @@ -0,0 +1,23 @@ +# Test data description + +This is the content output of a Player build, made with Unity 6000.7.0b2. +The diagnostic switch to enable TypeTrees was enabled when the build was performed. + +It is a build of the same project as `PlayerWithTypeTrees` (Unity 6000.0.65f1), with the same two +scenes, so the two folders can be compared file by file. `PlayerWithTypeTrees` is SerializedFile +version 22 and this one is version 26, which makes the pair the reference for the Unity 6.7 format +changes: independently versioned TypeTrees, the `[SerializeReference]` registry frame, and shared +subtrees. For a version 23 comparison use `AssetBundleTypeTreeVariations/v23_Inline`. + +## Content + +See `../PlayerWithTypeTrees/README.md` for the scenes, the sharing arrangement and the scripting +types - they are the same here. + +`resources.assets` is also included, which the version 23 folder does not have. +The `globalgamemanagers.assets.resS` splash screen data is not checked in, because it is 2.8 MB in +this build and nothing needs it. + +## BuildReport + +The LastBuild.buildreport file (created in the Library folder) has also been copied in. diff --git a/TestCommon/Data/PlayerWithTypeTreesV26/globalgamemanagers b/TestCommon/Data/PlayerWithTypeTreesV26/globalgamemanagers new file mode 100644 index 0000000..0b4af2b Binary files /dev/null and b/TestCommon/Data/PlayerWithTypeTreesV26/globalgamemanagers differ diff --git a/TestCommon/Data/PlayerWithTypeTreesV26/globalgamemanagers.assets b/TestCommon/Data/PlayerWithTypeTreesV26/globalgamemanagers.assets new file mode 100644 index 0000000..947bb22 Binary files /dev/null and b/TestCommon/Data/PlayerWithTypeTreesV26/globalgamemanagers.assets differ diff --git a/TestCommon/Data/PlayerWithTypeTreesV26/level0 b/TestCommon/Data/PlayerWithTypeTreesV26/level0 new file mode 100644 index 0000000..dc95564 Binary files /dev/null and b/TestCommon/Data/PlayerWithTypeTreesV26/level0 differ diff --git a/TestCommon/Data/PlayerWithTypeTreesV26/level1 b/TestCommon/Data/PlayerWithTypeTreesV26/level1 new file mode 100644 index 0000000..7931251 Binary files /dev/null and b/TestCommon/Data/PlayerWithTypeTreesV26/level1 differ diff --git a/TestCommon/Data/PlayerWithTypeTreesV26/resources.assets b/TestCommon/Data/PlayerWithTypeTreesV26/resources.assets new file mode 100644 index 0000000..d14608c Binary files /dev/null and b/TestCommon/Data/PlayerWithTypeTreesV26/resources.assets differ diff --git a/TestCommon/Data/PlayerWithTypeTreesV26/sharedassets0.assets b/TestCommon/Data/PlayerWithTypeTreesV26/sharedassets0.assets new file mode 100644 index 0000000..9486a21 Binary files /dev/null and b/TestCommon/Data/PlayerWithTypeTreesV26/sharedassets0.assets differ diff --git a/TestCommon/Data/PlayerWithTypeTreesV26/sharedassets0.assets.resS b/TestCommon/Data/PlayerWithTypeTreesV26/sharedassets0.assets.resS new file mode 100644 index 0000000..84d803c --- /dev/null +++ b/TestCommon/Data/PlayerWithTypeTreesV26/sharedassets0.assets.resS @@ -0,0 +1 @@ + !!!!!!!!!!"""""""""""""##########$$$$$$$$$$%%%% !!!!!!!!!!"""""""""""""#########$$$$$$$$$$$%%%%% !!!!!!!!!!"""""""""""""#########$$$$$$$$$$%%%%%%% !!!!!!!!!!""""""""""""#########$$$$$$$$$$%%%%%%%%% !!!!!!!!!!""""""""""""#########$$$$$$$$$$%%%%%%%%%% !!!!!!!!!!""""""""""""#########$$$$$$$$$%%%%%%%%%%%% !!!!!!!!!""""""""""""########$$$$$$$$$$%%%%%%%%%%%&& !!!!!!!!!""""""""""""########$$$$$$$$$%%%%%%%%%%%&&&& !!!!!!!!!!"""""""""""########$$$$$$$$$%%%%%%%%%%%&&&&& !!!!!!!!!!""""""""""########$$$$$$$$$%%%%%%%%%%%&&&&&&& !!!!!!!!!!""""""""""########$$$$$$$$$%%%%%%%%%%&&&&&&&&& !!!!!!!!!!""""""""""########$$$$$$$$%%%%%%%%%%&&&&&&&&&&& !!!!!!!!!!""""""""""#######$$$$$$$$$%%%%%%%%%%&&&&&&&&&&&& !!!!!!!!!!"""""""""########$$$$$$$$%%%%%%%%%%&&&&&&&&&&&&'' !!!!!!!!!!"""""""""########$$$$$$$$%%%%%%%%%%&&&&&&&&&&&'''' !!!!!!!!!!"""""""""#######$$$$$$$$%%%%%%%%%%&&&&&&&&&&&'''''' !!!!!!!!!"""""""""#######$$$$$$$$%%%%%%%%%&&&&&&&&&&&'''''''' !!!!!!!!!""""""""########$$$$$$$$%%%%%%%%%&&&&&&&&&&'''''''''' !!!!!!!!!""""""""#######$$$$$$$$%%%%%%%%%&&&&&&&&&&&''''''''''' !!!!!!!!!""""""""#######$$$$$$$$%%%%%%%%%&&&&&&&&&&''''''''''''( !!!!!!!!""""""""########$$$$$$$%%%%%%%%%&&&&&&&&&&''''''''''''((( !!!!!!!!!""""""""#######$$$$$$$$%%%%%%%%&&&&&&&&&&''''''''''''((((( !!!!!!!!!""""""""########$$$$$$$$%%%%%%%%&&&&&&&&&&'''''''''''((((((( !!!!!!!!!!""""""""########$$$$$$$%%%%%%%%&&&&&&&&&&'''''''''''((((((((( !!!!!!!!!!""""""""########$$$$$$$$%%%%%%%%&&&&&&&&&'''''''''''((((((((()) !!!!!!!!!!!""""""""########$$$$$$$%%%%%%%%&&&&&&&&&'''''''''''((((((((())))  !!!!!!!!!!""""""""########$$$$$$$$%%%%%%%%&&&&&&&&&''''''''''(((((((()))))))  !!!!!!!!!!""""""""########$$$$$$$$%%%%%%%%&&&&&&&&&''''''''''(((((((()))))))))  !!!!!!!!!!"""""""""########$$$$$$$$%%%%%%%&&&&&&&&&'''''''''(((((((())))))))))** !!!!!!!!!!!!""""""""########$$$$$$$$%%%%%%%%&&&&&&&&'''''''''(((((((()))))))))***** !!!!!!!!!!!!!"""""""""########$$$$$$$$%%%%%%%&&&&&&&&&''''''''(((((((()))))))))*******!!!!! !!!!!!!!!!!!!"""""""""########$$$$$$$$%%%%%%%%&&&&&&&&''''''''((((((()))))))))**********!!!!!!!!!!!!!!!! !!!!!!!!!!!!!!"""""""""########$$$$$$$$%%%%%%%%&&&&&&&&'''''''(((((((())))))))**********+++!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"""""""""#########$$$$$$$$%%%%%%%%&&&&&&&'''''''((((((()))))))))*********++++++""!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"""""""""#########$$$$$$$$%%%%%%%%&&&&&&&'''''''((((((())))))))*********+++++++++"""""""!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!""""""""""""""########$$$$$$$$%%%%%%%%&&&&&&''''''''((((((())))))))*********++++++++,,,""""""""""""""!!!!!!!!!!!!!!!!!!!!!"""""""""""""""""#########$$$$$$$$%%%%%%%&&&&&&&'''''''((((((())))))))*********++++++++,,,,,,####""""""""""""""""""""""""""""""""""""""""""""###########$$$$$$$$%%%%%%%&&&&&&&'''''''((((((()))))))*********+++++++,,,,,,,,,-##########""""""""""""""""""""""""""""""""""###########$$$$$$$$$$%%%%%%%&&&&&&&'''''''((((((()))))))********+++++++,,,,,,,,,----$#################""""""""""""""""""""##############$$$$$$$$$$%%%%%%%&&&&&&&'''''''((((((()))))))********+++++++,,,,,,,,--------$$$$$$$##########################################$$$$$$$$$$%%%%%%%%&&&&&&&'''''''((((((()))))))*******+++++++,,,,,,,,---------..$$$$$$$$$$$$$################################$$$$$$$$$$%%%%%%%%%&&&&&&&'''''''((((((()))))))*******+++++++,,,,,,,---------......%%%%$$$$$$$$$$$$$$$$$$$$$$##########$$$$$$$$$$$$$$$$%%%%%%%%%&&&&&&&&'''''''((((((())))))*******++++++,,,,,,,,---------.........%%%%%%%%%%$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$%%%%%%%%%%%&&&&&&&'''''''((((((())))))******+++++++,,,,,,,---------..........///&%%%%%%%%%%%%%%%%%$$$$$$$$$$$$$$$$$$$$$%%%%%%%%%%%%%%%%&&&&&&&&&'''''''(((((())))))******+++++++,,,,,,,--------..........///////&&&&&&&&%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%&&&&&&&&&&''''''''((((((())))))******+++++++,,,,,,,--------........./////////00&&&&&&&&&&&&&&%%%%%%%%%%%%%%%%%%%%%%%%%%&&&&&&&&&&&&&&&''''''''((((((()))))))******+++++++,,,,,,,-------........./////////000000''''''''&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&''''''''''(((((((())))))*******++++++,,,,,,,--------......./////////00000000001'''''''''''''''&&&&&&&&&&&&&&&&&&&&&&&&&&&'''''''''''''(((((((()))))))*******++++++,,,,,,,-------.......////////0000000001111111((((((('''''''''''''''''''''''''''''''''''''''''''((((((((())))))))******+++++++,,,,,,,------.......////////00000000111111111112((((((((((((((((''''''''''''''''''''''''''((((((((((((())))))))*******+++++++,,,,,,------.......///////0000000001111111112222222)))))))))(((((((((((((((((((((((((((((((((((((((())))))))))*******+++++++,,,,,,-------......///////00000000111111111222222222233**)))))))))))))))))))(((((((((((((((((())))))))))))))*********+++++++,,,,,,-------......///////000000011111111122222222233333333************))))))))))))))))))))))))))))))))))***********++++++++,,,,,,-------......///////0000000111111122222222223333333333344+++++**********************************************+++++++++,,,,,,,-------......//////000000011111112222222223333333333344444444+++++++++++++++++***********************++++++++++++++,,,,,,,,-------.......//////0000000111111122222222333333333444444444444555,,,,,,,,,+++++++++++++++++++++++++++++++++++++,,,,,,,,,,,--------......///////00000011111112222222233333333444444444445555555555---,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,---------........//////0000000111111222222233333333344444444455555555555555666------------------,,,,,,,,,,,,,,,,,,,,,--------------.........///////00000011111112222222333333344444444455555555555566666666666.........-----------------------------------............////////0000001111111222222233333334444444455555555555666666666666667777////............................................//////////0000000011111122222223333333444444455555555556666666666677777777777777//////////////////////////////////////////////////000000000111111112222222333333344444455555555566666666667777777777777788888888000000000000000000//////////////////000000000000000011111111122222223333333444444455555555666666666777777777778888888888888888891111111111111000000000000000000000000001111111111111122222222233333334444444555555556666666677777777778888888888888899999999999911111111110000000000000000000000000000000000000000000011111111111111222222222223333333333444444444455555555555555566666666666666///////////////..................------,,,,,,,,,,,,,+++++++++++++++++++++++,,,,,,,,,,,,,------............////////////0000000000**)))))))(((((((''''''&&&&&&%%%%%%$$$$##"""""""!!!!!! !!!!!""""""""##$$$%%%%%%%&&&&&&'''''((((((()))))))***""""!!!!!  !!!!""""" !!!!!""""""#####$$$$$%%% !!!!!""""""#####$$$$$%%%% !!!!!""""""#####$$$$$%%%%% !!!!""""""####$$$$$%%%%%%& !!!!!""""#####$$$$%%%%%%&&& !!!!!""""####$$$$$%%%%%&&&&& !!!!!""""####$$$$%%%%%&&&&&&' !!!!!""""####$$$$%%%%%&&&&&''' !!!!"""""####$$$%%%%%&&&&&&'''' !!!!!""""####$$$$%%%%%&&&&&'''''( !!!!""""####$$$$%%%%&&&&&'''''((( !!!!"""""###$$$$%%%%&&&&&'''''((((( !!!!!""""####$$$$%%%%&&&&'''''((((())  !!!!!""""#####$$$%%%%&&&&&''''((((()))) !!!!!"""""####$$$$%%%%&&&&''''(((()))))**!! !!!!!!!""""####$$$$%%%%&&&&''''(((()))))****!!!!!!!!!!!!!!!!!!!!!!!!"""""#####$$$%%%%&&&&''''(((())))****+++""""!!!!!!!!!!!!!!!!!""""""#####$$$$%%%%&&&'''(((()))))****++++,#"""""""""""""""""""""""#####$$$$$%%%&&&''''(((())))****+++,,,,,########"""""""""""#######$$$$$%%%%&&&''''((())))****+++,,,,,---$$$$$#################$$$$$$%%%%&&&&'''(((()))****+++,,,,,----..%%%%$$$$$$$$$$$$$$$$$$$$$%%%%&&&&''''((())))***+++,,,,----...../&&&&%%%%%%%%%%%%%%%%%%%%%&&&&&''''((()))****+++,,,,----..../////''&&&&&&&&&&&&&&%&&&&&&&&&''''(((()))****+++,,,----....////00000((''''''''''''&'&'''''''''((((()))***++++,,,----...////000001111))((((((((((((((((((((((()))))****+++,,,----...////0000111112222***))))))))))))))))))))))*****+++,,,,---...////00001112222223333++++********************+++++,,,,---...////000111122223333334444,,,,,,+++++++++++++++,,,,,,,----...///00001111222333334444445555--------,,,,,,,,,,,--------....///000011122223333344444555555555.-..------------------......///000001112222333344444455555555566--------------,-------------......//0000011111122223333333344444+++++************)))))))))))))*******+++++,,,,-----.....////////&&&%%%%%%%%$$$$########""""""""""""####$##$$$$%%%&&&&&''''((((()  ! !!!""###$$$%% !!!""###$$%%%& !!!""##$$$%%%&& !!!""##$$%%%&&&' !!!""##$$%%&&&''( !!""##$$$%%&&''((( !!!""##$$%%&&''((())!!!!!!!!!!!!!""##$$%%&&''((())**"""""!!!"""""##$$$%%&''(())***++######""#####$$%%&&''(())**++,,,%$$$$$$$$$$$%%%&&''(()**++,,,--.&&&%%%%%%%%&&&''(())**+,,,--.../'''''''''''''(())**++,,--..////0((((((((((()))**++,,--..///00000))))))))))))***++,,--...///00000(((((((((((()))***++,,,----.....%%%$$$$$$$$$$$%%%%&&'''((()))))*  !!!"""##$$ !"##$%% !!"#$$%&& !!"#$%%&'(!!!!!!"##$%&'(()"""""##$%&''()**$$$$$$%%&'()**++$$$%%%&''()**+++$$$$$%%&&'(()))*!!!!!!!""##$$%%& ! !"#$%!!!"#$&'"""#%&''!!"#$%&& !"# !#$ !#$ !          ! !! !!! !!!! !!!!!! !!!!!!! !!!!!!!!! !!!!!!!!!! ""!!!!!!!!!  """"!!!!!!!!!  """"""!!!!!!!!!  """"""""!!!!!!!!  !!!#"""""""""!!!!!!!!  !!!!!!##""""""""""!!!!!!!!  !!!!!!!!!!####""""""""""!!!!!!!!  !!!!!!!!!""""######""""""""""!!!!!!!!  !!!!!!!!!!"""""""########""""""""""!!!!!!!!  !!!!!!!!!""""""""""#$#########"""""""""!!!!!!!!  !!!!!!!!!!""""""""""####$$$$########""""""""""!!!!!!!  !!!!!!!!!""""""""""#######$$$$$$$#########"""""""""!!!!!!!!  !!!!!!!!!""""""""""########$$$$%%$$$$$$$########"""""""""!!!!!!!!!!  !!!!!!!!!""""""""""########$$$$$$%%%%%%$$$$$$$########"""""""""!!!!!!!!!!!  !!!!!!!!!!"""""""""########$$$$$$$%%%%%%%%%%%%$$$$$$$########"""""""""!!!!!!!!!!!!  !!!!!!!!!!!!"""""""""########$$$$$$$%%%%%%%&&&&%%%%%%%%$$$$$$$#######""""""""""!!!!!!!!!!!!! !!!!!!!!!!!!!""""""""""########$$$$$$$%%%%%%%&&&&&&&&&&&%%%%%%%$$$$$$$$########""""""""""!!!!!!!!!!!!!!!!!!!! !!!!!!!!!!!!!!!!!!"""""""""""#######$$$$$$$%%%%%%%%&&&&&&&'''&&&&&&&&%%%%%%%$$$$$$$$#########""""""""""!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!""""""""""""########$$$$$$$%%%%%%%&&&&&&&&''''''''''&&&&&&&&%%%%%%%$$$$$$$$$#########"""""""""""""!!!!!!!!!!!!!!!!!!!!!"""""""""""""""#########$$$$$$$%%%%%%%%&&&&&&&'''''''''(((''''''&&&&&&&%%%%%%%%%$$$$$$$$$##########""""""""""""""""""""""""""""""""""""""#########$$$$$$$$%%%%%%%%&&&&&&&&''''''''(((((((('''''''''&&&&&&&%%%%%%%%%%$$$$$$$$$##############""""""""""""""""""""""###########$$$$$$$$$%%%%%%%%&&&&&&&&''''''''((((((((())))(((''''''''&&&&&&&&&%%%%%%%%%%$$$$$$$$$$$$#################################$$$$$$$$$$%%%%%%%%%&&&&&&&&''''''''((((((((())))))))*((((((('''''''&&&&&&&&&&%%%%%%%%%%$$$$$$$$$$$$$$$$$$#############$$$$$$$$$$$$$$%%%%%%%%%&&&&&&&&&''''''''((((((((()))))))*******))((((((((''''''''&&&&&&&&&&&%%%%%%%%%%%%%%$$$$$$$$$$$$$$$$$$$$$$$$$$$%%%%%%%%%%%%&&&&&&&&&'''''''''(((((((()))))))********+++++)))))(((((((((''''''''''&&&&&&&&&&%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%&&&&&&&&&&&''''''''''((((((())))))))*******+++++++++,,))))))))))(((((((('''''''''''&&&&&&&&&&&&&&&%%%%%%%%%%%%%%%%%%%%%%&&&&&&&&&&&&&'''''''''(((((((()))))))********+++++++++,,,,,,,,*****)))))))))(((((((((('''''''''''&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&'''''''''''((((((((()))))))********++++++++,,,,,,,,,------*********))))))))))((((((((((('''''''''''''''''''&&&&''''''''''''''''''''((((((((()))))))))*******+++++++++,,,,,,,,--------.....+++++**********)))))))))))(((((((((((((('''''''''''''''''''''''(((((((((((()))))))))********++++++++,,,,,,,---------.........///++++++++++***********)))))))))))))((((((((((((((((((((((((((((((())))))))))))********++++++++,,,,,,,--------.........//////////0,,,,,+++++++++++*************)))))))))))))))))))))))))))))))))))))))*********++++++++,,,,,,,,--------......../////////0000000000,,,,,,,,,,,+++++++++++++********************************************+++++++++,,,,,,,,--------......../////////000000000011111111------,,,,,,,,,,,,++++++++++++++++++******************++++++++++++++,,,,,,,,,,--------........////////00000000111111111122222222--------------,,,,,,,,,,,,,,,,+++++++++++++++++++++++++++,,,,,,,,,,,,---------........///////00000000111111111222222222223333333.......-----------------,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-----------........////////0000000011111111222222222233333333333344444/...............------------------------------------------...........////////000000001111111122222222333333333334444444444445555////////////...............................................//////////00000000111111122222222333333333444444444455555555555555666000000///////////////////////////......../////////////////00000000011111111222222233333333444444444555555555556666666666666667770000000000000000000000////////////////////000000000000000111111111222222233333333444444455555555556666666666667777777777777777881111111111111110000000000000000000000000001111111111111122222222233333334444444555555555666666666777777777777888888888888888888911111111111100000000000000000000000000000000000000000000111111111111112222222222233333333333444444444445555555555555555566666666////////////////.................------,,,,,,,,,,,,,++++++++++++++++++++++++,,,,,,,,,,,,,--------............/////////////000000**))))))))((((((''''''&&&&&&%%%%%%$$$$##"""""""!!!!!! !!!!!"""""""##$$$$%%%%%%%&&&&&&''''''(((((())))))))*""""!!!!!  !!!!""""     ! !! !!!! !!!!! ""!!!!!  """"!!!!!  !#"""""!!!!  !!!!!###"""""!!!!  !!!!!"""$####"""""!!!!  !!!!""""###$$$####"""""!!!!  !!!!!""""####$$%%$$$$####""""!!!!!  !!!!!"""""####$$$$%%%%%%$$$$#####""""!!!!!! !!!!!!"""""####$$$$%%%&&&&&&%%%%$$$$$####"""""!!!!!!!!!!!!!!!!!!!"""""####$$$$%%%%%&&&'''''&&&&%%%%$$$$$#####""""""""""""""""""""#####$$$$%%%%&&&&''''((((''''&&&&&%%%%$$$$$$##################$$$$$%%%%%&&&'''''(((())))(((((''''&&&&&%%%%%%$$$$$$$$$$$$$$$$$%%%%&&&&&''''((((())))****+))))((((''''''&&&&&&%%%%%%%%%%%%%%&&&&&&''''((((())))***+++++,,,***))))))(((((''''''''&&&&&&&&''''''''(((()))))***++++,,,,,-----+++******)))))((((((((((((((((((((()))))****++++,,,,----.....///,,,+++++++*******))))))))))))*******+++++,,,,----..../////000000---,,,,,,,++++++++++++++++++++++,,,,,----.....////00001111112222..---------,,,,,,,,,,,,,,,,,------....////0000011112222223333333.......-------------------.......////000011112222233333334444444.--.----------,--,-,,,---------...../////00000011112222222233333++++++***********)))))))))))))))))******+++++,,,,-----........//&&&%%%%%%%%$$$$#####""#""""""""""""#"######$$$$%%%%&&&''''''((((   ! !! "!!!  """!!  !!##"""!!  !!"""$$##"""!!  !!!""##$$%%$$###""!!!! !!!!""###$$%%&&&%%%$$###"""""""""###$$$%%&&''''''&&&%%%$$$$$$$$$$%%%&&'''(()))(((('''&&&&&&%&&&&&'''(())***+++))))(((((''''''(((())***+++,,,,-)))))))(((((())))***+++,,,,-----(((((((((((((((()))****++++,,,,,%%%%%$$$$$$$$$$$$%%%%&&&''''((((!  !!!"""# ! "!  #"!!  !""$##""!!!!!!""#$$%%$$###"###$%%&&%%%%$$$$%%%&&'''$$$$$$$$$%%&&&&&"!!! !!!"""###! "!  !#"!!!!""""!!"""# !  !!!!!!!!!!!!!!!!!!!!"""""""""""""""########################$$$$$$$$$$$$$$$$$$%%%% !!!!!!!!!!!!!!!!!!!"""""""""""""""#########################$$$$$$$$$$$$$$$$$%%% !!!!!!!!!!!!!!!!!!!"""""""""""""""#########################$$$$$$$$$$$$$$$$$% !!!!!!!!!!!!!!!!!!""""""""""""""""########################$$$$$$$$$$$$$$$$$ !!!!!!!!!!!!!!!!!!""""""""""""""""########################$$$$$$$$$$$$$$$ !!!!!!!!!!!!!!!!!"""""""""""""""""########################$$$$$$$$$$$$$ !!!!!!!!!!!!!!!!""""""""""""""""""#######################$$$$$$$$$$$$ !!!!!!!!!!!!!!!!""""""""""""""""""#######################$$$$$$$$$$ !!!!!!!!!!!!!!!!"""""""""""""""""""#######################$$$$$$$$ !!!!!!!!!!!!!!!!"""""""""""""""""""######################$$$$$$$ !!!!!!!!!!!!!!!!"""""""""""""""""""#######################$$$$$ !!!!!!!!!!!!!!!""""""""""""""""""""######################$$$$ !!!!!!!!!!!!!!!!"""""""""""""""""""######################$$ !!!!!!!!!!!!!!!!!"""""""""""""""""""###################### !!!!!!!!!!!!!!!!""""""""""""""""""""#################### !!!!!!!!!!!!!!!!""""""""""""""""""""################## !!!!!!!!!!!!!!!!"""""""""""""""""""""################ !!!!!!!!!!!!!!!!""""""""""""""""""""""############# !!!!!!!!!!!!!!!!""""""""""""""""""""""########### !!!!!!!!!!!!!!!!"""""""""""""""""""""""######### !!!!!!!!!!!!!!!!"""""""""""""""""""""""####### !!!!!!!!!!!!!!!""""""""""""""""""""""""##### !!!!!!!!!!!!!!!!""""""""""""""""""""""""### !!!!!!!!!!!!!!!!""""""""""""""""""""""""" !!!!!!!!!!!!!!!!""""""""""""""""""""""" !!!!!!!!!!!!!!!!""""""""""""""""""""" !!!!!!!!!!!!!!!!!""""""""""""""""""" !!!!!!!!!!!!!!!!!""""""""""""""""" !!!!!!!!!!!!!!!!!!""""""""""""""" !!!!!!!!!!!!!!!!!!""""""""""""" !!!!!!!!!!!!!!!!!!""""""""""" !!!!!!!!!!!!!!!!!!""""""""" !!!!!!!!!!!!!!!!!!""""""" !!!!!!!!!!!!!!!!!!!""""" !!!!!!!!!!!!!!!!!!!""" !!!!!!!!!!!!!!!!!!!" !!!!!!!!!!!!!!!!!! !!!!!!!!!!!!!!!! !!!!!!!!!!!!!!! !!!!!!!!!!!!! !!!!!!!!!!! !!!!!!!!! !!!!!!!! !!!!!! !!!!! !!! !          !!!!!!!!!!"""""""###########$$$$$$$$$$%% !!!!!!!!!""""""""###########$$$$$$$$$$ !!!!!!!!!""""""""###########$$$$$$$$ !!!!!!!!"""""""""###########$$$$$$ !!!!!!!"""""""""############$$$$ !!!!!!!!""""""""""##########$$$ !!!!!!!""""""""""###########$ !!!!!!!!""""""""""########## !!!!!!!!""""""""""######## !!!!!!!!!""""""""""###### !!!!!!!!""""""""""""### !!!!!!!!""""""""""""# !!!!!!!!!""""""""""" !!!!!!!!!""""""""" !!!!!!!!!""""""" !!!!!!!!!""""" !!!!!!!!!""" !!!!!!!!!!" !!!!!!!!! !!!!!!! !!!!! !!! !      !!!!!""""#####$$$$$%% !!!!!""""#####$$$$$ !!!!"""""#####$$$ !!!!"""""#####$ !!!!""""##### !!!!"""""### !!!!"""""# !!!!"""" !!!!!"" !!!!! !!! !    !!"""##$$$% !!""###$$ !!""### !!""# !!"" !! !  !!""#$$ !""# !" !  !"# ! !"    !!!!  !!!!!!!!  "!!!!!!!!!!!  """""!!!!!!!!!!!  """""""""!!!!!!!!!!!  !!!#"""""""""""""!!!!!!!!!!  !!!!!!######""""""""""""!!!!!!!!!!!  !!!!!!!!!!$#########"""""""""""""!!!!!!!!!!!  !!!!!!!!!!!!""$$$$$###########""""""""""""!!!!!!!!!!!!  !!!!!!!!!!!!!"""""%%$$$$$$$$###########""""""""""""!!!!!!!!!!!!!  !!!!!!!!!!!!!!"""""""""%%%%%%%$$$$$$$$############""""""""""""!!!!!!!!!!!!!! !!!!!!!!!!!!!!!"""""""""""###&&&%%%%%%%%%$$$$$$$$#############"""""""""""""!!!!!!!!!!!!!!!! !!!!!!!!!!!!!!!!!!!!!!"""""""""""########&&&&&&&&&%%%%%%%%%$$$$$$$$#############"""""""""""""""!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"""""""""""############$''''&&&&&&&&&&%%%%%%%%%$$$$$$$$$##############"""""""""""""""""!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!""""""""""""""############$$$$$$''''''''''&&&&&&&&&&%%%%%%%%%$$$$$$$$$$################""""""""""""""""""""""""""""""""""""""""""""""""#############$$$$$$$$$$$$((((('''''''''''&&&&&&&&&&&%%%%%%%%%$$$$$$$$$$###################"""""""""""""""""""""""""""""#################$$$$$$$$$$$$$%%%%(((((((((((('''''''''''&&&&&&&&&&&%%%%%%%%%%$$$$$$$$$$$$##############################################$$$$$$$$$$$$$$$$%%%%%%%%%%))))))(((((((((((((''''''''''''&&&&&&&&&&%%%%%%%%%%%$$$$$$$$$$$$$$$###############$$$$###$$$$$$$$$$$$$$$$$$$$$$$%%%%%%%%%%%%%%%&*)))))))))))))(((((((((((((''''''''''''&&&&&&&&&&&%%%%%%%%%%%%%$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$%%%%%%%%%%%%%%%%%%%%&&&&&&&&**********)))))))))))))((((((((((((''''''''''''&&&&&&&&&&&&%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%&&&&&&&&&&&&&&+++++++************)))))))))))))(((((((((((('''''''''''''&&&&&&&&&&&&&&&&&&&&&&&&&&%%%%%%%%%%%%%%%&&&&&&&&&&&&&&&&&&&&&&&''''''',,,++++++++++++++***********)))))))))))))(((((((((((((''''''''''''''''&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&'''''''''''''''',,,,,,,,,,,,,,+++++++++++++***********)))))))))))))((((((((((((((('''''''''''''''''''''''''''''''''''''''''''''''''''''''(((((((----------,,,,,,,,,,,,,,,++++++++++++***********))))))))))))))((((((((((((((((('''''''''''''''''''''''''((((((((((((((((((((((((........---------------,,,,,,,,,,,,,++++++++++++************))))))))))))))))(((((((((((((((((((((((((((((((((((((((())))))))))))//////................-------------,,,,,,,,,,,,,++++++++++++**************)))))))))))))))))))))))))))))))))))))))))))))))))))***00///////////////////..............------------,,,,,,,,,,,,++++++++++++++*******************************************************000000000000000000000///////////////............-----------,,,,,,,,,,,,,++++++++++++++++++++++++*******************+++++++++++++1111111111111111111000000000000000000////////////............------------,,,,,,,,,,,,,,,,,++++++++++++++++++++++++++++++++++++++222222222222222222221111111111111111100000000000000///////////...........---------------,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,333333333333333333333222222222222222222111111111111100000000000///////////..............----------------------------------------4444444444444444444433333333333333333333332222222222221111111111100000000000////////////....................--------------------5555555555555555555555444444444444444444444433333333333332222222222111111111100000000000///////////////.........................66666666666666666666666655555555555555555555554444444444443333333333222222222111111111100000000000000///////////////////////////777777777777777777777777777777766666666666666666665555555555544444444433333333332222222221111111111110000000000000000000000/////888888888888888888888888888888888888888777777777777777666666666655555555544444444333333333222222222221111111111111111100000000009999999999999999999999999999999999999999999888888888888887777777777666666655555555544444444333333333322222222222222111111111111166666666666666666666666666655555555555555555555554444444444444433333333333333333222222222222222222222111111111111111111111111111000000/////////////............-------,,,,,,,,,,,,,++++++++++++++++++++++++++,,,,,,,,,,,,-------.................///////////////*))))))))((((((''''''&&&&&&%%%%%%$$$$$##""""""!!!!!! !!!!!"""""""##$$$$%%%%%%%&&&&&&'''''((((((()))))))**""""!!!!  !!!!!"""" !  !!!!!  """"!!!!!!  !!####"""""!!!!!  !!!!!$$$#####""""""!!!!!!  !!!!!!"""%%%$$$$######""""""!!!!!!!! !!!!!!!!""""""#&&&%%%%%$$$$$######"""""""!!!!!!!!!!!!!!!!!!!!!!!!!!""""""######''''&&&&&%%%%%$$$$$$#######""""""""""""""""""""""""""######$$$$$(((((''''''&&&&&%%%%%$$$$$$$########################$$$$$$$$%%%%)))))))((((('''''&&&&&&%%%%%%%$$$$$$$$$$$$$$$$$$$$$%%%%%%%%%%&&&++*******)))))((((((''''''&&&&&&&&&%%%%%%%%%%%%%%%&%&&&&&&&&&&'',,,,,++++++*******)))))))((((((''''''''''''''''&&'''''''''''''((.--------,,,,,,,++++++*******)))))))((((((((((((((((((((((((()))///////.......-------,,,,,,,++++++*********)*))))))))))))*******1000000000000/////////.....------,,,,,,,++++++++++++++++++++++++222222222222111111111100000///////.....--------,,,,,,,,,,,,,,,,,33334343333333333333222222221111100000//////..........----------444444444444444444444444333333222221111100000////////...........3333333333333333333322222222111110000000//////./.........--..---///./..........-.-----,,,,,,,,,,+++++++++*++*+++++++++++++++++++(((('''''''&&&&%%%%$$$%$$$$$#################$$$$$$%%%%%%%%%&&&&  !!!  #"""!!!  !!$$###"""!!! !!!"""&%%%$$$###""""!!!!!!!!!!"""""###('''&&&%%%%$$$#############$$$$%)))))(((''''&&&%%%%%%%%%%%%%%&&&+++++*****)))(((('''''''''''''''-----,,,,,,++++****)))))((((((((---.....------,,,++++****))))))),,,,,,,,,,,,,,++++***)))))((((((((((((((('''''&&&&&%%%%%%%%%%%%%##""""!!!!    ""!!  !!$$###""!!!!!!"""&&&%%%$$$#####$$'''''''&&%%%%%$$'''''''&&%%%$$$$$#####""""!!!!!!!  #"""!!!!####""!!  %%%%%%%$$$$$$$$$$$$$$$$$#######################"""""""""""""""!!!!!!!!!!!!!!!!!!!!! %%%%%%%%%$$$$$$$$$$$$$$$$$$######################"""""""""""""""!!!!!!!!!!!!!!!!!!!! %%%%%%%%%%%%$$$$$$$$$$$$$$$$$$#####################"""""""""""""""!!!!!!!!!!!!!!!!!!!! %%%%%%%%%%%%%%$$$$$$$$$$$$$$$$$$####################"""""""""""""""!!!!!!!!!!!!!!!!!!!! %%%%%%%%%%%%%%%%%$$$$$$$$$$$$$$$$$$###################"""""""""""""""!!!!!!!!!!!!!!!!!!! &%%%%%%%%%%%%%%%%%%%$$$$$$$$$$$$$$$$$$#################""""""""""""""""!!!!!!!!!!!!!!!!!!! &&&&%%%%%%%%%%%%%%%%%%%$$$$$$$$$$$$$$$$$#################""""""""""""""""!!!!!!!!!!!!!!!!!! &&&&&&&%%%%%%%%%%%%%%%%%%$$$$$$$$$$$$$$$$$$################""""""""""""""""!!!!!!!!!!!!!!!!! &&&&&&&&&&%%%%%%%%%%%%%%%%%%$$$$$$$$$$$$$$$$$################""""""""""""""""!!!!!!!!!!!!!!!!! &&&&&&&&&&&&&%%%%%%%%%%%%%%%%%$$$$$$$$$$$$$$$$$###############""""""""""""""""!!!!!!!!!!!!!!!!!!!!! &&&&&&&&&&&&&&&&%%%%%%%%%%%%%%%%$$$$$$$$$$$$$$$$$###############""""""""""""""""!!!!!!!!!!!!!!!!!!!!!!!! &&&&&&&&&&&&&&&&&&&%%%%%%%%%%%%%%%$$$$$$$$$$$$$$$$$###############""""""""""""""""!!!!!!!!!!!!!!!!!!!!!!!!!! '&&&&&&&&&&&&&&&&&&&&&&%%%%%%%%%%%%%$$$$$$$$$$$$$$$$$###############""""""""""""""""!!!!!!!!!!!!!!!!!!!!!!!!!!! ''''&&&&&&&&&&&&&&&&&&&&&&%%%%%%%%%%%%$$$$$$$$$$$$$$$$$##############""""""""""""""""!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ''''''''&&&&&&&&&&&&&&&&&&&&&%%%%%%%%%%%%$$$$$$$$$$$$$$$$##############"""""""""""""""""!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ''''''''''''&&&&&&&&&&&&&&&&&&&%%%%%%%%%%%%$$$$$$$$$$$$$$$$##############""""""""""""""""""""!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ''''''''''''''''&&&&&&&&&&&&&&&&&%%%%%%%%%%%%$$$$$$$$$$$$$$$$##############""""""""""""""""""""""!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!''''''''''''''''''''&&&&&&&&&&&&&&&&%%%%%%%%%%%$$$$$$$$$$$$$$$$#############""""""""""""""""""""""""""!!!!!!!!!!!!!!!!!!!!!!!!!!''''''''''''''''''''''''&&&&&&&&&&&&&&%%%%%%%%%%%%$$$$$$$$$$$$$$$##############""""""""""""""""""""""""""!!!!!!!!!!!!!!!!!!!!!!!((''''''''''''''''''''''''''&&&&&&&&&&&&%%%%%%%%%%%%%$$$$$$$$$$$$$$###############"""""""""""""""""""""""""""!!!!!!!!!!!!!!!!!!!((((((('''''''''''''''''''''''&&&&&&&&&&&&&%%%%%%%%%%%%$$$$$$$$$$$$$$###############""""""""""""""""""""""""""""!!!!!!!!!!!!!!!!(((((((((((('''''''''''''''''''''&&&&&&&&&&&&&%%%%%%%%%%%%$$$$$$$$$$$$$$#################"""""""""""""""""""""""""""!!!!!!!!!!!!((((((((((((((((''''''''''''''''''''&&&&&&&&&&&&%%%%%%%%%%%%%$$$$$$$$$$$$$$##################""""""""""""""""""""""""""!!!!!!!!!((((((((((((((((((((('''''''''''''''''&&&&&&&&&&&&&%%%%%%%%%%%%$$$$$$$$$$$$$$$####################""""""""""""""""""""""""""""""))))(((((((((((((((((((((('''''''''''''''&&&&&&&&&&&&&%%%%%%%%%%%%$$$$$$$$$$$$$$$####################"""""""""""""""""""""""""""))))))))((((((((((((((((((((((''''''''''''''&&&&&&&&&&&&&%%%%%%%%%%%%$$$$$$$$$$$$$$$####################""""""""""""""""""""""""))))))))))))(((((((((((((((((((((''''''''''''''&&&&&&&&&&&&&%%%%%%%%%%%%$$$$$$$$$$$$$$$####################"""""""""""""""""""""))))))))))))))))((((((((((((((((((((''''''''''''''&&&&&&&&&&&&&%%%%%%%%%%%%$$$$$$$$$$$$$$#######################""""""""""""""""*****)))))))))))))))(((((((((((((((((((''''''''''''''&&&&&&&&&&&&&%%%%%%%%%%%%$$$$$$$$$$$$$$####################################**********)))))))))))))))(((((((((((((((((('''''''''''''&&&&&&&&&&&&%%%%%%%%%%%%%$$$$$$$$$$$$$$$################################**************)))))))))))))))))((((((((((((((('''''''''''''&&&&&&&&&&&&%%%%%%%%%%%%%$$$$$$$$$$$$$$$#############################+******************)))))))))))))))((((((((((((((('''''''''''''&&&&&&&&&&&%%%%%%%%%%%%%$$$$$$$$$$$$$$$$$$$$$$####################+++++++*****************))))))))))))))(((((((((((((('''''''''''''&&&&&&&&&&&%%%%%%%%%%%%%$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$#########++++++++++++*****************)))))))))))))((((((((((((((''''''''''''&&&&&&&&&&&%%%%%%%%%%%%%%$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$++++++++++++++++++****************))))))))))))((((((((((((('''''''''''&&&&&&&&&&&%%%%%%%%%%%%%%%%%%$$$$$$$$$$$$$$$$$$$$$$$$$$$$$,,,,,,++++++++++++++++++**************))))))))))))(((((((((((('''''''''''&&&&&&&&&&&&&&%%%%%%%%%%%%%%%%%%%%%$$$$$$$$$$$$$$$$$$$$,,,,,,,,,,,,++++++++++++++++++*************)))))))))))(((((((((((''''''''''''&&&&&&&&&&&&&&&%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%--,,,,,,,,,,,,,,,++++++++++++++++++************)))))))))))(((((((((((''''''''''''&&&&&&&&&&&&&&&&%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%---------,,,,,,,,,,,,,,,++++++++++++++++************))))))))))((((((((((((''''''''''''&&&&&&&&&&&&&&&&%%%%%%%%%%%%%%%%%%%%%%%%%%----------------,,,,,,,,,,,,,,++++++++++++++************))))))))))(((((((((((('''''''''''''&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&.....-----------------,,,,,,,,,,,,,,+++++++++++++***********)))))))))))((((((((((('''''''''''''&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&.............----------------,,,,,,,,,,,,+++++++++++++**********)))))))))))(((((((((('''''''''''''''''''''&&&&&&&&&&&&&&&&&&&&&&....................---------------,,,,,,,,,,,++++++++++++**********))))))))))((((((((((((((''''''''''''''''''''''''''''''''''''/////////...................-------------,,,,,,,,,,+++++++++++**********)))))))))))))(((((((((((((((((''''''''''''''''''''''''''///////////////////................------------,,,,,,,,,++++++++++************)))))))))))))(((((((((((((((((((((((''''''''''''''0000///////////////////////..............-----------,,,,,,,,,,++++++++++************)))))))))))))))(((((((((((((((((((((((((((((0000000000000000///////////////////............-----------,,,,,,,,,,++++++++++***********))))))))))))))))))(((((((((((((((((((((11110000000000000000000000////////////////...........----------,,,,,,,,,,+++++++++++************))))))))))))))))))))))))))))))))11111111111111111100000000000000000/////////////..........----------,,,,,,,,,,+++++++++++****************)))))))))))))))))))))))222111111111111111111111111100000000000000////////////..........---------,,,,,,,,,,++++++++++++**************************)))))))2222222222222222222211111111111111111000000000000//////////..........---------,,,,,,,,,,,++++++++++++++++***********************33333322222222222222222222222222111111111111100000000000/////////.........----------,,,,,,,,,,,,+++++++++++++++++++++++++*******33333333333333333333333322222222222222222111111111111000000000/////////.........-----------,,,,,,,,,,,,,,,++++++++++++++++++++++44444444443333333333333333333333333322222222222221111111111000000000/////////.........-------------,,,,,,,,,,,,,,,,,,,,,,,,+++++44444444444444444444444444444333333333333333322222222222111111111000000000/////////..........---------------,,,,,,,,,,,,,,,,,,,,55555555555555555554444444444444444444444333333333333222222222211111111000000000/////////............--------------------------,55555555555555555555555555555555555544444444444444333333333322222222211111111000000000//////////...............-----------------66666666666666666666666666555555555555555555555444444444433333333322222222211111111000000000////////////........................666666666666666666666666666666666666666666555555555555544444444433333333222222221111111110000000000///////////////..............77777777777777777777777777777777777766666666666666665555555555444444443333333322222222111111111000000000000/////////////////////777777777777777888888777777777777777777777777777666666666665555555554444444433333333222222221111111111100000000000000000////////888888888888888888888888888888888888888888888777777777777666666666555555554444444433333333222222222211111111111110000000000000009999999999999999999999999999999999999999988888888888888777777777766666665555555554444444333333333222222222221111111111111111110099999999999999999::::::::::::::::99999999999999999999888888888877777777666666665555555544444444333333333322222222222222111111111666666666666666666666666666666666666665555555555555555544444444444443333333333333333222222222222222222221111111111111111111111110000000000////////////.............------,,,,,,,,,,,,,,++++++++++++++++++++,,,,,,,,,,,,,------...............///////////////////***))))))))(((((('''''&&&&&&&%%%%%%$$$##""""""""!!!!! !!!!!""""""""##$$$%%%%%%%&&&&&&'''''((((((())))))))**"""""!!!!  !!!!!""""%%%%%$$$$$$$$$##########""""""""!!!!!!!!!! %%%%%%$$$$$$$$$$##########""""""""!!!!!!!!! %%%%%%%%%$$$$$$$$$#########""""""""!!!!!!!!!! &&&%%%%%%%%%$$$$$$$$$########""""""""!!!!!!!!! &&&&&&%%%%%%%%$$$$$$$$$########""""""""!!!!!!!!!! &&&&&&&&&%%%%%%%%$$$$$$$$########""""""""!!!!!!!!!!!! ''&&&&&&&&&&&%%%%%%$$$$$$$$#######"""""""""!!!!!!!!!!!!!!! '''''&&&&&&&&&&%%%%%%$$$$$$$$$######""""""""""!!!!!!!!!!!!!!!! '''''''''&&&&&&&&%%%%%%%$$$$$$$########"""""""""""!!!!!!!!!!!!!!''''''''''''&&&&&&&&%%%%%%$$$$$$$########""""""""""""""!!!!!!!!!((((((''''''''''&&&&&&&%%%%%%$$$$$$$########""""""""""""""!!!!!!(((((((((((''''''''&&&&&&%%%%%%%$$$$$$##########"""""""""""""""!))))((((((((((''''''''&&&&&&%%%%%%$$$$$$$$#########""""""""""""")))))))((((((((((('''''''&&&&&&%%%%%%$$$$$$$$############"""""""****)))))))))((((((((''''''&&&&&&%%%%%%%$$$$$$$$################********)))))))))(((((((''''''&&&&&&%%%%%%%$$$$$$$$$############+++++*********))))))(((((((''''''&&&&&&%%%%%%%%$$$$$$$$$$$$$$$$$,,,+++++++********)))))))((((((''''''&&&&&&%%%%%%%%%%$$$$$$$$$$$,,,,,,,,++++++++*******)))))(((((('''''''&&&&&&&%%%%%%%%%%%%%%%%------,,,,,,,,+++++++******))))))((((('''''''&&&&&&&&&&&&&&&&&&&....---------,,,,,,,++++++*****))))))(((((''''''''''&&&&&&&&&&&&//...........------,,,,,,+++++******))))))((((((((''''''''''''''///////////........------,,,,,+++++******)))))))((((((((((((((((00000000000////////......-----,,,,,++++++******)))))))))))))((((11111111111000000000/////.....-----,,,,,++++++***********)))))))222222222222111111111000000////.....-----,,,,,+++++++++*********33333333333333222222221111100000/////....------,,,,,,,,,++++++++44444444444444443333333222222111110000////......-------,,,,,,,,,5555555555555554554444444433333222111110000//////......---------5666666666666666665555555544444333322222111100000//////.........6566666666666666666666655555554443333322221111000000////////....444444444444444444444343333332222221111100000////////...........////////////./.......-------,,,,,,,,++++++++++++++++++++++++++++)((((((('''&&&&&&%%%%%%$$$$$$#$$###$###$###$$$$$$$$%%%%%%%&%&&&&!!  %%%%$$$$$####"""""!!!!! &&%%%%%$$$$####""""!!!!!! &&&&&%%%%$$$$####""""!!!!!!!! '''&&&&%%%%$$$$####"""""!!!!!!!!('''''&&&&%%%%$$$####""""""""!!!(((((''''&&&&%%%$$$$#####"""""""))))((((('''&&&%%%%$$$$#########****))))(((('''&&&%%%%$$$$$$$$##+++++****)))((('''&&&&%%%%%%$$$$,,,,,,++++***)))((('''&&&&&&%%%%..-----,,,,+++***)))(((''''''''&////......---,,,++***))))(((((((0000000/////...--,,,++****))))))0111111100000///..---,,+++*****)000111111110000//..---,,+++*****....//////.....---,,+++***)))))(*****))))))(((((''''&&&&&&&&%%%%$$####""""!!!! !!&%%$$$##"""!!!!!'&&&%%$$##""""!!('''&&%%$$###""")))((''&&%%$$$##****))((''&&%%%$++++++**)(('&&&%+,,,,,++*))(''&&*******))(''&&%%&&&%%%%$$###""""! %%$$#""!''&%$$#"((('&%$#'''&&$#"##""! $$#"$$#! !  \ No newline at end of file diff --git a/TestCommon/Data/PlayerWithTypeTreesV26/sharedassets1.assets b/TestCommon/Data/PlayerWithTypeTreesV26/sharedassets1.assets new file mode 100644 index 0000000..0e0e3c6 Binary files /dev/null and b/TestCommon/Data/PlayerWithTypeTreesV26/sharedassets1.assets differ diff --git a/TextDumper/TextDumperTool.cs b/TextDumper/TextDumperTool.cs index 8b135d5..e886517 100644 --- a/TextDumper/TextDumperTool.cs +++ b/TextDumper/TextDumperTool.cs @@ -22,6 +22,8 @@ public class TextDumperTool // Set during the processed of each Serialized File UnityFileReader m_Reader; + // End of the object currently being dumped; bounds the registry frame read. + long m_ObjectEnd; SerializedFile m_SerializedFile; public enum DumpFormat @@ -82,7 +84,7 @@ public int Dump(DumpOptions options) int DumpSerializedFile() { - if (ReportIfMissingTypeTrees(m_Options.Path, m_Options.Path)) + if (ReportIfNotDumpable(m_Options.Path, m_Options.Path)) return 1; try @@ -111,11 +113,20 @@ int DumpSerializedFile() // dump needs TypeTrees to interpret object data, so a SerializedFile without them cannot be dumped. // Detecting this up front avoids handing the file to the native loader, which would otherwise emit - // misleading version mismatch errors or crash the process. Returns true (and prints a clear message) - // when the file has no TypeTrees. The path may be a real file or an entry in a mounted archive. - bool ReportIfMissingTypeTrees(string path, string displayName) + // misleading version mismatch errors or crash the process. The same goes for a version this build + // does not know: the native loader reports it as a file that may be corrupt, which sends the reader + // looking for the wrong problem. Returns true (and prints a clear message) when the file cannot be + // dumped. The path may be a real file or an entry in a mounted archive. + bool ReportIfNotDumpable(string path, string displayName) { using var stream = new UnityFileStream(path); + + if (SerializedFileDetector.IsVersionUnsupported(stream, out var versionError)) + { + Console.Error.WriteLine($"Error: \"{displayName}\" cannot be dumped. {versionError}"); + return true; + } + if (!SerializedFileDetector.IsMissingTypeTrees(stream)) return false; @@ -158,7 +169,7 @@ int DumpArchive() var node2 = singleSerializedFile.Value; Console.Error.WriteLine($"Processing {node2.Path} {node2.Size} {node2.Flags}"); - if (ReportIfMissingTypeTrees("/" + node2.Path, node2.Path)) + if (ReportIfNotDumpable("/" + node2.Path, node2.Path)) return 1; m_Writer = Console.Out; OutputSerializedFile("/" + node2.Path); @@ -172,7 +183,7 @@ int DumpArchive() if (node.Flags.HasFlag(ArchiveNodeFlags.SerializedFile)) { - if (ReportIfMissingTypeTrees("/" + node.Path, node.Path)) + if (ReportIfNotDumpable("/" + node.Path, node.Path)) { anyMissingTypeTrees = true; continue; @@ -227,9 +238,10 @@ void OutputSerializedFile(string path) continue; var offset = obj.Offset; + m_ObjectEnd = obj.Offset + obj.Size; m_Writer.Write($"ID: {obj.Id} (ClassID: {obj.TypeId}) "); - RecursiveDump(root, ref offset, 0); + RecursiveDump(root, ref offset, 0, isRootObject: true); m_Writer.WriteLine(); dumpedObject = true; } @@ -244,7 +256,7 @@ void OutputSerializedFile(string path) } } - void RecursiveDump(TypeTreeNode node, ref long offset, int level, int arrayIndex = -1) + void RecursiveDump(TypeTreeNode node, ref long offset, int level, int arrayIndex = -1, bool isRootObject = false) { bool skipChildren = false; @@ -328,6 +340,11 @@ void RecursiveDump(TypeTreeNode node, ref long offset, int level, int arrayIndex { foreach (var child in node.Children) { + // A version 3 registry sits in the data before this field, with no node of its + // own. Only a root object carries one, hence isRootObject. + if (isRootObject && child.HasSerializedRefs) + DumpManagedReferenceFrame(ref offset, level + 1); + RecursiveDump(child, ref offset, level + 1); } } @@ -470,6 +487,57 @@ void DumpManagedReferenceRegistry(TypeTreeNode node, ref long offset, int level) } } + // Shaped like the version 1 and 2 dumps above, so the three stay comparable in a diff. + void DumpManagedReferenceFrame(ref long offset, int level) + { + var registry = ManagedReferenceRegistry.ReadFrame(m_Reader, offset, m_ObjectEnd - offset); + + WriteIndentedLine(level, "references (ManagedReferenceRegistry)"); + WriteIndentedLine(level + 1, $"version (int) {registry.Version}"); + + foreach (var entry in registry.Entries) + { + WriteIndentedLine(level + 1, $"rid({entry.Rid}) ReferencedObject"); + + if (entry.IsNull) + { + WriteIndentedLine(level + 2, "null"); + continue; + } + + WriteIndentedLine(level + 2, "type (ReferencedManagedType)"); + WriteIndentedLine(level + 3, $"class (string) {entry.ClassName}"); + WriteIndentedLine(level + 3, $"ns (string) {entry.Namespace}"); + WriteIndentedLine(level + 3, $"asm (string) {entry.AssemblyName}"); + WriteIndentedLine(level + 2, "data ReferencedObjectData "); + + var dataOffset = entry.DataOffset; + DumpReferencedObjectData(entry.ClassName, entry.Namespace, entry.AssemblyName, ref dataOffset, level + 3); + } + + offset += registry.FrameSize; + } + + void WriteIndentedLine(int level, string text) + { + AppendIndent(level); + m_StringBuilder.Append(text); + m_Writer.WriteLine(m_StringBuilder); + m_StringBuilder.Clear(); + } + + // A referenced object's data is laid out by its own type tree rather than the containing + // object's, and that tree's root stands for the instance itself, so only its fields are dumped. + void DumpReferencedObjectData(string className, string namespaceName, string assemblyName, ref long offset, int level) + { + var refTypeRoot = m_SerializedFile.GetRefTypeTypeTreeRoot(className, namespaceName, assemblyName); + + foreach (var child in refTypeRoot.Children) + { + RecursiveDump(child, ref offset, level); + } + } + bool DumpManagedReferenceData(TypeTreeNode refTypeNode, TypeTreeNode referencedTypeDataNode, ref long offset, int level, long id) { if (refTypeNode.Children.Count < 3) @@ -516,13 +584,7 @@ bool DumpManagedReferenceData(TypeTreeNode refTypeNode, TypeTreeNode referencedT return true; } - var refTypeRoot = m_SerializedFile.GetRefTypeTypeTreeRoot(className, namespaceName, assemblyName); - - // Dump the ReferencedObject using its own TypeTree, but skip the root. - foreach (var child in refTypeRoot.Children) - { - RecursiveDump(child, ref offset, level + 1); - } + DumpReferencedObjectData(className, namespaceName, assemblyName, ref offset, level + 1); return true; } diff --git a/UnityBinaryFormat/SerializedFileDetector.cs b/UnityBinaryFormat/SerializedFileDetector.cs index 4c5a312..122c824 100644 --- a/UnityBinaryFormat/SerializedFileDetector.cs +++ b/UnityBinaryFormat/SerializedFileDetector.cs @@ -114,6 +114,14 @@ public class TypeTreeInfo // version >= 20) // ----------------------------------------------------------------------- + /// + /// Format version stamped into the inline TypeTree blob (version >= 23, which is where the blob + /// gained its prefix). Up to and including 23 the stamp repeats the SerializedFile version; + /// from 24 (kIndependentTypeTreeVersion) TypeTrees are versioned independently, starting at 32. + /// 0 when there is no inline blob to read it from. + /// + public uint TypeTreeFormatVersion { get; set; } + /// /// C# class name of the SerializeReference type. /// string.Empty for regular (non-ref) type entries. @@ -163,6 +171,30 @@ public class ScriptType public long PathID { get; set; } } +/// +/// One entry of the shared subtree table (version >= 26, kSharedSubtreeSupport). Each entry is a +/// compound subtree the file stores once, keyed by the hash of its content, that the file's +/// TypeTrees reference in place of repeating it. +/// +public class SharedSubtreeInfo +{ + /// + /// Hash identifying the subtree's content. Also the key used to fetch it from an external + /// TypeTree store when the blob is not inline. + /// + public UnityHash128 ContentHash { get; set; } + + /// + /// Size in bytes of the inline blob. 0 means the blob was extracted to an external store. + /// + public uint SerializedSize { get; set; } + + /// + /// True when the blob is stored in this file rather than an external TypeTree store. + /// + public bool Inline => SerializedSize > 0; +} + /// /// Information extracted from the beginning of a Unity SerializedFile metadata section. /// @@ -214,6 +246,16 @@ public class SerializedFileMetadata /// Null until the metadata section has been parsed. /// public ExternalReference[] ExternalReferences { get; set; } + + /// + /// Number of shared subtree entries. Always 0 for files with version < 26. + /// + public int SharedSubtreeCount { get; set; } + + /// + /// Summary of each shared subtree entry. Empty array for files with version < 26. + /// + public SharedSubtreeInfo[] SharedSubtrees { get; set; } = Array.Empty(); } /// @@ -259,9 +301,10 @@ public static class SerializedFileDetector // Older files have format differences that we do not attempt to support. private const uint MinMetadataParseVersion = 19; - // Maximum version for metadata section parsing (kExtractedTypeTreeSupport = 23, Unity 6000.4). + // Maximum version for metadata section parsing (kSharedSubtreeSupport = 26, Unity 6000.7). // Files newer than this version may have an unknown format and cannot be parsed safely. - private const uint MaxMetadataParseVersion = 23; + // Public so that callers and tests can report or check the ceiling without repeating the number. + public const uint MaxMetadataParseVersion = 26; // Reasonable version range for SerializedFiles // Unity versions currently use values in the 20s-30s range @@ -280,12 +323,20 @@ public static class SerializedFileDetector private const uint SupportsRefObjectVersion = 20; // m_RefTypes list (appears after externals) private const uint StoresTypeDependenciesVersion = 21; // Per-type dependency list added private const uint ExtractedTypeTreeSupportVersion = 23; // TypeTree blob may be extracted externally + // 24 and 25 have no constant here because they leave the metadata layout alone: 24 moves the + // TypeTree blob onto its own version number space and 25 moves the [SerializeReference] registry + // into the object's data, neither of which this parser reads. + private const uint SharedSubtreeSupportVersion = 26; // Shared subtree table follows m_RefTypes // Per-type-entry constants private const int MonoBehaviourClassID = 114; // persistentTypeID for MonoBehaviour private const int UndefinedPersistentTypeID = -1; // persistentTypeID for types with no known ClassID private const uint TypeTreeNodeSize = 32; // Bytes per node in the blob (version >= 18) + // A TypeTree blob with the version >= 23 prefix starts [uint32 'tthm'][uint32 version]. + private const uint TypeTreeBlobMagic = 0x7474686D; // 'tthm', stored as this uint32 value + private const int TypeTreeBlobPrefixSize = 8; + /// /// Attempts to detect if a file is a Unity SerializedFile by reading and validating its header. /// Returns false immediately if the file doesn't match the expected format. @@ -655,6 +706,19 @@ public static bool TryParseMetadata(Stream stream, SerializedFileInfo headerInfo } } + /// + /// Reports whether the stream is a SerializedFile whose version this parser cannot read. + /// Returns true only when the version is known and outside the supported range, so a stream + /// that is not a SerializedFile at all is left for the caller's usual handling. + /// + public static bool IsVersionUnsupported(Stream stream, out string errorMessage) + { + errorMessage = null; + + return TryDetectSerializedFile(stream, out var fileInfo) + && !IsMetadataVersionSupported(fileInfo.Version, out errorMessage); + } + /// /// Returns true when the stream is a SerializedFile we can positively confirm has no TypeTrees. /// Returns false for files that have TypeTrees and for anything we cannot parse (so callers fall @@ -771,6 +835,34 @@ private static void ParseExtendedMetadata(BinaryReader reader, SerializedFileInf for (int i = 0; i < refTypeCount; i++) refTypeTrees[i] = ReadTypeEntry(reader, version, swap, isRefType: true, enableTypeTree); metadata.SerializedReferenceTypeTrees = refTypeTrees; + + if (version < SharedSubtreeSupportVersion) + return; + + // --- Shared subtree table (version >= 26) --- + // Per-entry layout: + // [Hash128 contentHash] + // [uint32 blobSize] (0 = blob extracted to an external TypeTree store) + // [blob] (blobSize bytes, absent when the blob was extracted) + int subtreeCount = BinaryFileHelper.ReadInt32(reader, swap); + metadata.SharedSubtreeCount = subtreeCount; + + var subtrees = new SharedSubtreeInfo[subtreeCount]; + for (int i = 0; i < subtreeCount; i++) + { + var contentHash = BinaryFileHelper.ReadHash128(reader, swap); + uint blobSize = BinaryFileHelper.ReadUInt32(reader, swap); + + if (blobSize > 0) + stream.Seek(blobSize, SeekOrigin.Current); + + subtrees[i] = new SharedSubtreeInfo + { + ContentHash = contentHash, + SerializedSize = blobSize, + }; + } + metadata.SharedSubtrees = subtrees; } catch { @@ -868,7 +960,15 @@ private static TypeTreeInfo ReadTypeEntry(BinaryReader reader, uint version, boo // Version >= 23 with inline blob: skip exactly typeTreeSize bytes. // The blob starts with its own 8-byte magic+version prefix, followed by // node count, char count, node array, and string buffer. - stream.Seek(typeTreeSize, SeekOrigin.Current); + var blobStart = stream.Position; + + if (typeTreeSize >= TypeTreeBlobPrefixSize && + BinaryFileHelper.ReadUInt32(reader, swap) == TypeTreeBlobMagic) + { + info.TypeTreeFormatVersion = BinaryFileHelper.ReadUInt32(reader, swap); + } + + stream.Seek(blobStart + typeTreeSize, SeekOrigin.Begin); } info.InlineTypeTree = true; } diff --git a/UnityDataTool.Tests/AnalyzeExitCodeTests.cs b/UnityDataTool.Tests/AnalyzeExitCodeTests.cs index b5b3b7b..458c438 100644 --- a/UnityDataTool.Tests/AnalyzeExitCodeTests.cs +++ b/UnityDataTool.Tests/AnalyzeExitCodeTests.cs @@ -1,8 +1,11 @@ +using System; +using System.Buffers.Binary; using System.IO; using System.Linq; using System.Threading.Tasks; using Microsoft.Data.Sqlite; using NUnit.Framework; +using UnityDataTools.BinaryFormat; namespace UnityDataTools.UnityDataTool.Tests; @@ -94,4 +97,30 @@ public async Task Analyze_SomeFilesFailed_SucceedsAndKeepsDatabase() StringAssert.DoesNotContain(NothingAnalyzedMessage, output); Assert.That(File.Exists(databasePath), Is.True); } + + // A file from a newer Unity than this build understands used to surface as whatever went wrong + // first, which sent the reader looking for a corrupt file. The version is known before the file + // is opened, so it is reported (issue #130). + [Test] + public async Task Analyze_VersionNewerThanSupported_ReportsTheVersion() + { + var newerFile = Path.Combine(m_TestOutputFolder, "future.assets"); + var bytes = File.ReadAllBytes(Path.Combine(TestContext.CurrentContext.TestDirectory, + "Data", "PlayerWithTypeTreesV26", "sharedassets1.assets")); + + // Derived rather than hardcoded so this does not churn as Unity adds versions. It has to + // stay inside the range the detector considers plausible for a SerializedFile header, + // otherwise the file is not recognised as one at all and analyze ignores it instead. + var unsupportedVersion = SerializedFileDetector.MaxMetadataParseVersion + 1; + + // The version is a big-endian uint32 at offset 8 of the header. + BinaryPrimitives.WriteUInt32BigEndian(bytes.AsSpan(8), unsupportedVersion); + File.WriteAllBytes(newerFile, bytes); + + var (exitCode, output) = await RunAnalyze(newerFile, "-o", SQLTestHelper.GetDatabasePath(m_TestOutputFolder)); + + Assert.AreEqual(1, exitCode); + StringAssert.Contains($"version {unsupportedVersion}", output); + StringAssert.Contains($"supports up to version {SerializedFileDetector.MaxMetadataParseVersion}", output); + } } diff --git a/UnityDataTool.Tests/AnalyzeV26Tests.cs b/UnityDataTool.Tests/AnalyzeV26Tests.cs new file mode 100644 index 0000000..bcf837d --- /dev/null +++ b/UnityDataTool.Tests/AnalyzeV26Tests.cs @@ -0,0 +1,81 @@ +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Data.Sqlite; +using NUnit.Framework; + +namespace UnityDataTools.UnityDataTool.Tests; + +#pragma warning disable NUnit2005, NUnit2006 + +// From SerializedFile version 25 the [SerializeReference] registry is a frame in the object's data +// that no TypeTree node describes, so analyze has to step over it to read the object's own fields +// and walk into it to find the references its instances hold. +public class AnalyzeV26Tests +{ + private string m_TestOutputFolder; + private string m_ManagedReferencesBundle; + + [OneTimeSetUp] + public void OneTimeSetup() + { + m_TestOutputFolder = Path.Combine(TestContext.CurrentContext.TestDirectory, "v26_test_folder"); + m_ManagedReferencesBundle = Path.Combine(TestContext.CurrentContext.TestDirectory, + "Data", "AssetBundleTypeTreeVariations", "v26", "managedreferences.bundle"); + Directory.CreateDirectory(m_TestOutputFolder); + Directory.SetCurrentDirectory(m_TestOutputFolder); + } + + [TearDown] + public void Teardown() + { + SqliteConnection.ClearAllPools(); + var testDir = new DirectoryInfo(m_TestOutputFolder); + testDir.EnumerateFiles().ToList().ForEach(f => f.Delete()); + testDir.EnumerateDirectories().ToList().ForEach(d => d.Delete(true)); + } + + async Task Analyze(string input) + { + var databasePath = SQLTestHelper.GetDatabasePath(m_TestOutputFolder); + Assert.AreEqual(0, await Program.Main(new[] { "analyze", input, "-o", databasePath })); + return SQLTestHelper.OpenDatabase(databasePath); + } + + [Test] + public async Task Analyze_Version26_FindsReferencesInsideTheRegistry() + { + using var db = await Analyze(m_ManagedReferencesBundle); + + // A PPtr held by a [SerializeReference] instance, whose data is laid out by that instance's + // own type tree rather than the object's. + SQLTestHelper.AssertQueryInt(db, + "SELECT COUNT(*) FROM refs_view WHERE property_path LIKE 'references.rid(%).data.material' AND property_type = '$Material'", + 1, "PPtr held by a [SerializeReference] instance"); + } + + // In a version 26 file a PPtr is usually reached through a shared subtree, which is a node with + // a byte size and no children of its own - the shape of a basic type. A reader that takes it at + // face value lands in the right place afterwards and simply never sees the reference, so the + // reference count is what catches it. The version 22 build of the same scene says what to + // expect. + [TestCase("PlayerWithTypeTrees")] + [TestCase("PlayerWithTypeTreesV26")] + public async Task Analyze_FindsTheSceneReferences_WhicheverFormat(string folder) + { + var level0 = Path.Combine(TestContext.CurrentContext.TestDirectory, "Data", folder, "level0"); + + using var db = await Analyze(level0); + + SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM objects", 7, "objects in the scene"); + SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM refs_view", 13, "references in the scene"); + + SQLTestHelper.AssertQueryInt(db, + "SELECT COUNT(*) FROM refs_view WHERE property_path = 'm_GameObject'", 3, + "component back-references to their GameObject"); + + SQLTestHelper.AssertQueryInt(db, + "SELECT COUNT(*) FROM refs_view WHERE property_path LIKE 'm_Component%'", 3, + "GameObject references to its components"); + } +} diff --git a/UnityDataTool.Tests/DumpTests.cs b/UnityDataTool.Tests/DumpTests.cs index e5bab66..5a5ccf3 100644 --- a/UnityDataTool.Tests/DumpTests.cs +++ b/UnityDataTool.Tests/DumpTests.cs @@ -315,6 +315,97 @@ public async Task Dump_Stdout_AssetBundle_SerializationDemo_ContainsExpectedFiel Assert.That(output, Does.Not.Contain("293, 294, 295, 296,")); } + static async Task DumpToString(params string[] args) + { + using var sw = new StringWriter(); + var currentOut = Console.Out; + try + { + Console.SetOut(sw); + Assert.AreEqual(0, await Program.Main(args)); + } + finally + { + Console.SetOut(currentOut); + } + + return sw.ToString(); + } + + // The same asset built by Unity 6.0 and 6.7. From SerializedFile version 25 its + // [SerializeReference] registry is a frame in the object's data rather than a TypeTree node, so + // the two dumps are the reference for that change: the registry version differs, the instance + // it holds does not. + [TestCase("PlayerWithTypeTrees", 2)] + [TestCase("PlayerWithTypeTreesV26", 3)] + public async Task Dump_Stdout_SerializeReference_ReadsRegistryWhicheverVersion(string folder, int expectedRegistryVersion) + { + var path = Path.Combine(m_TestDataFolder, folder, "sharedassets1.assets"); + + var output = await DumpToString("dump", path, "--stdout", "--type", "MonoBehaviour"); + + Assert.That(output, Does.Contain("m_Name (string) ScriptableObjectWIthSerializeReference")); + Assert.That(output, Does.Contain($"version (int) {expectedRegistryVersion}")); + Assert.That(output, Does.Contain("rid(6911265806470873295) ReferencedObject")); + Assert.That(output, Does.Contain("class (string) Data")); + Assert.That(output, Does.Contain("ns (string) MyNamespace")); + Assert.That(output, Does.Contain("Info (string) Some info")); + Assert.That(output, Does.Contain("Flag (UInt8) 1")); + + // The field pointing at the instance carries its rid. A reader that does not step over the + // registry frame reads the frame header here instead, which is a plausible-looking number + // rather than an outright failure. + Assert.That(output, Does.Contain("rid (SInt64) 6911265806470873295")); + } + + // Unity's own version 26 fixture, which carries the [SerializeReference] shapes the rest of the + // test data does not. See TestCommon/Data/AssetBundleTypeTreeVariations/README.md. + [Test] + public async Task Dump_Stdout_Version26_ReadsEveryRegistryShape() + { + var path = Path.Combine(m_TestDataFolder, "AssetBundleTypeTreeVariations", "v26", "managedreferences.bundle"); + + var output = await DumpToString("dump", path, "--stdout", "--type", "MonoBehaviour"); + + // A null reference: a record with no type and no data, which only the version 3 registry + // can express. + Assert.That(output, Does.Contain("rid(-2) ReferencedObject")); + Assert.That(output, Does.Contain("empty (managedReference)")); + + // A PPtr inside a referenced object's data. + Assert.That(output, Does.Contain("class (string) ManagedReferenceTestBehaviour/TexturedShape")); + Assert.That(output, Does.Contain("material (PPtr<$Material>)")); + + // A reference nested inside another, and a collection of references. + Assert.That(output, Does.Contain("class (string) ManagedReferenceTestBehaviour/GroupedShape")); + Assert.That(output, Does.Contain("inner (managedReference)")); + Assert.That(output, Does.Contain("Array[4]")); + + // A [Serializable] class with no fields is a compound of no size, not an unresolvable one. + Assert.That(output, Does.Contain("noData (ShapeNoData)")); + + // The plain fields either side of the references. A reader that does not step over the + // registry frame reads these shifted rather than failing, so they are the real check. + Assert.That(output, Does.Contain("before (int) 11")); + Assert.That(output, Does.Contain("after (int) 22")); + } + + // Shared subtrees (version 26) replace a repeated compound with a single node that has a byte + // size and no children of its own - the shape of a basic type. A reader that takes them at face + // value drops the compound's fields silently, so the check is that they are still dumped. + [Test] + public async Task Dump_Stdout_Version26_ExpandsSharedSubtrees() + { + var path = Path.Combine(m_TestDataFolder, "PlayerWithTypeTreesV26", "level0"); + + var output = await DumpToString("dump", path, "--stdout", "--type", "Transform"); + + Assert.That(output, Does.Contain("m_LocalPosition (Vector3f)")); + Assert.That(output, Does.Contain("m_GameObject (PPtr)")); + Assert.That(output, Does.Contain("m_FileID (int)")); + Assert.That(output, Does.Contain("m_PathID (SInt64)")); + } + // The expected bit patterns are the well-known IEEE 754 representations of the // SerializationDemo field values (also verified against python struct.pack). // Runs under a comma-decimal locale to confirm the output is culture-invariant. diff --git a/UnityFileSystem.Tests/SerializedFileV26Tests.cs b/UnityFileSystem.Tests/SerializedFileV26Tests.cs new file mode 100644 index 0000000..45fe413 --- /dev/null +++ b/UnityFileSystem.Tests/SerializedFileV26Tests.cs @@ -0,0 +1,213 @@ +using System.IO; +using System.Linq; +using NUnit.Framework; +using UnityDataTools.FileSystem; +using UnityDataTools.FileSystem.TypeTreeReaders; + +namespace UnityDataTools.FileSystem.Tests; + +// Unity 6.7 (SerializedFile version 26) changes how a type tree and an object's data are laid out: +// a compound the file shares between types appears as a single node standing in for it, and the +// [SerializeReference] registry moves into the object's data as a frame no node describes. +// +// The fixtures are two builds of the same project, PlayerWithTypeTrees (version 22) and +// PlayerWithTypeTreesV26, so each test can check that the two report the same content. +public class SerializedFileV26Tests +{ + const int MonoBehaviourClassId = 114; + + string m_V22Folder; + string m_V26Folder; + + [OneTimeSetUp] + public void OneTimeSetUp() + { + var data = Path.Combine(TestContext.CurrentContext.TestDirectory, "Data"); + m_V22Folder = Path.Combine(data, "PlayerWithTypeTrees"); + m_V26Folder = Path.Combine(data, "PlayerWithTypeTreesV26"); + + UnityFileSystem.Init(); + } + + [OneTimeTearDown] + public void OneTimeTearDown() + { + UnityFileSystem.Cleanup(); + } + + static ObjectInfo FindObjectOfType(SerializedFile sf, int classId) + { + var obj = sf.Objects.FirstOrDefault(o => o.TypeId == classId); + Assert.That(obj.Size, Is.Not.EqualTo(0), $"No object of ClassID {classId} in the file"); + return obj; + } + + static void ForEachNode(TypeTreeNode node, System.Action action) + { + action(node); + + foreach (var child in node.Children) + ForEachNode(child, action); + } + + [Test] + public void TypeTree_Version26_UsesSharedSubtreeReferences() + { + using var sf = UnityFileSystem.OpenSerializedFile(Path.Combine(m_V26Folder, "level0")); + + var refs = 0; + + foreach (var obj in sf.Objects) + { + ForEachNode(sf.GetTypeTreeRoot(obj.Id), node => + { + if (!node.IsSharedSubtreeRef) + return; + + refs++; + + // The failure this guards against is silent: a shared subtree reference has a byte + // size and no children of its own, which is how a basic type is recognised, so a + // fixed-size compound would be read as a primitive of the same width and its + // fields - PPtrs among them - simply dropped. + Assert.That(node.IsBasicType, Is.False, $"{node.Name} ({node.Type}) read as a basic type"); + }); + } + + Assert.That(refs, Is.GreaterThan(0), "A version 26 file is expected to share its compound subtrees"); + } + + [Test] + public void TypeTree_Version26_ResolvesPPtrThroughSharedSubtree() + { + using var sf = UnityFileSystem.OpenSerializedFile(Path.Combine(m_V26Folder, "level0")); + + TypeTreeNode pptr = null; + + foreach (var obj in sf.Objects) + { + ForEachNode(sf.GetTypeTreeRoot(obj.Id), node => + { + if (pptr == null && node.IsSharedSubtreeRef && node.Type.StartsWith("PPtr<")) + pptr = node; + }); + } + + Assert.That(pptr, Is.Not.Null, "Expected a PPtr shared through a subtree"); + Assert.That(pptr.Children.Count, Is.EqualTo(2)); + Assert.That(pptr.Children[0].Name, Is.EqualTo("m_FileID")); + Assert.That(pptr.Children[1].Name, Is.EqualTo("m_PathID")); + } + + [Test] + public void TypeTree_Version22_HasNoSharedSubtreeReferences() + { + using var sf = UnityFileSystem.OpenSerializedFile(Path.Combine(m_V22Folder, "level0")); + + foreach (var obj in sf.Objects) + { + ForEachNode(sf.GetTypeTreeRoot(obj.Id), node => + Assert.That(node.IsSharedSubtreeRef, Is.False, $"{node.Name} ({node.Type}) in a version 22 file")); + } + } + + // Reads the one [SerializeReference] instance the fixture holds, whichever registry layout the + // file uses. The expected values are the same for both builds, since it is the same asset. + void AssertRegistryContents(string folder, int expectedVersion) + { + var path = Path.Combine(folder, "sharedassets1.assets"); + + using var sf = UnityFileSystem.OpenSerializedFile(path); + using var fileReader = new UnityFileReader(path, 1024 * 1024); + + var obj = FindObjectOfType(sf, MonoBehaviourClassId); + var reader = new RandomAccessReader(sf, sf.GetTypeTreeRoot(obj.Id), fileReader, obj.Offset); + + var registry = reader.Registry; + + Assert.That(registry, Is.Not.Null); + Assert.That(registry.Version, Is.EqualTo(expectedVersion)); + Assert.That(registry.Entries.Count, Is.EqualTo(1)); + + var entry = registry.Entries[0]; + + Assert.That(entry.IsNull, Is.False); + Assert.That(entry.ClassName, Is.EqualTo("Data")); + Assert.That(entry.Namespace, Is.EqualTo("MyNamespace")); + Assert.That(entry.AssemblyName, Is.EqualTo("Assembly-CSharp")); + + // The field that points at the instance stores only its rid. Reading the two consistently + // is what catches a reader that does not step over the registry frame: it would take the + // frame's first bytes as the rid rather than failing outright. + Assert.That(reader["reference"]["rid"].GetValue(), Is.EqualTo(entry.Rid)); + + // Fields after the registry must not be shifted either. + Assert.That(reader["m_Name"].GetValue(), Is.EqualTo("ScriptableObjectWIthSerializeReference")); + + var data = new RandomAccessReader(sf, + sf.GetRefTypeTypeTreeRoot(entry.ClassName, entry.Namespace, entry.AssemblyName), + fileReader, entry.DataOffset); + + Assert.That(data["Info"].GetValue(), Is.EqualTo("Some info")); + Assert.That(data["Flag"].GetValue(), Is.EqualTo(1)); + } + + [Test] + public void Registry_Version26_ReadsFrameEntries() + { + AssertRegistryContents(m_V26Folder, ManagedReferenceRegistry.FrameVersion); + } + + [Test] + public void Registry_Version22_ReadsNodeEntriesAsTheSameShape() + { + AssertRegistryContents(m_V22Folder, 2); + } + + // ManagedReferenceTestBehaviour carries a plain int either side of its [SerializeReference] + // fields, and a [Serializable] class with no fields at all. Reading the fields after those is + // what catches a reader that mis-sizes either: it reads them shifted rather than failing. + [Test] + public void Registry_Version26_ReadsFieldsAroundTheFrameAndAFieldLessCompound() + { + var path = Path.Combine(TestContext.CurrentContext.TestDirectory, "Data", + "AssetBundleTypeTreeVariations", "v26", "managedreferences.bundle"); + + using var archive = UnityFileSystem.MountArchive(path, "archive:/"); + var cab = "archive:/" + archive.Nodes.First(n => n.Flags.HasFlag(ArchiveNodeFlags.SerializedFile)).Path; + + using var sf = UnityFileSystem.OpenSerializedFile(cab); + using var fileReader = new UnityFileReader(cab, 1024 * 1024); + + var obj = sf.Objects.First(o => o.TypeId == MonoBehaviourClassId && o.Size > 500); + var reader = new RandomAccessReader(sf, sf.GetTypeTreeRoot(obj.Id), fileReader, obj.Offset, objectSize: obj.Size); + + // "before" is the field the frame precedes, so it is read past the whole frame. + Assert.That(reader["before"].GetValue(), Is.EqualTo(11)); + + // "after" follows both the reference fields and the field-less compound. + Assert.That(reader["noData"].Size, Is.EqualTo(0)); + Assert.That(reader["after"].GetValue(), Is.EqualTo(22)); + + var registry = reader.Registry; + Assert.That(registry.Version, Is.EqualTo(ManagedReferenceRegistry.FrameVersion)); + Assert.That(registry.Entries.Any(e => e.IsNull), Is.True, "Expected a null reference entry"); + Assert.That(registry.Entries.Count(e => e.ClassName.EndsWith("Shape")), Is.GreaterThan(1), + "Expected several instances of one type, sharing a Types table entry"); + } + + [Test] + public void Registry_ObjectWithoutReferences_IsNull() + { + var path = Path.Combine(m_V26Folder, "sharedassets1.assets"); + + using var sf = UnityFileSystem.OpenSerializedFile(path); + using var fileReader = new UnityFileReader(path, 1024 * 1024); + + // PreloadData holds no [SerializeReference] instances. + var obj = FindObjectOfType(sf, 150); + var reader = new RandomAccessReader(sf, sf.GetTypeTreeRoot(obj.Id), fileReader, obj.Offset); + + Assert.That(reader.Registry, Is.Null); + } +} diff --git a/UnityFileSystem/DllWrapper.cs b/UnityFileSystem/DllWrapper.cs index 9b074cf..13af51f 100644 --- a/UnityFileSystem/DllWrapper.cs +++ b/UnityFileSystem/DllWrapper.cs @@ -78,6 +78,8 @@ public enum ReturnCode ErrorCreatingArchiveFile, ErrorAddingFileToArchive, TypeNotFound, + HigherTypeTreeVersion, + RequiresSubtreeApi, } [Flags] @@ -136,6 +138,9 @@ public enum TypeTreeFlags IsManagedReference = 1 << 1, IsManagedReferenceRegistry = 1 << 2, IsArrayOfRefs = 1 << 3, + // The registry frame precedes this node's data when its type tree is used as a root. + HasSerializedRefs = 1 << 4, + IsSharedSubtreeRef = 1 << 5, } [Flags] @@ -168,6 +173,40 @@ public struct TypeTreeInfo public readonly string AssemblyName; } +// Parsed header of a [SerializeReference] registry frame. Every offset is relative to the frame's +// first byte, which is the byte passed to the GetRegistryFrame* calls as the frame data. +[StructLayout(LayoutKind.Sequential)] +public struct RegistryFrameInfo +{ + public int Version; + public int TypeCount; + public int RecordCount; + public ulong FrameSize; + public ulong BlobsOffset; +} + +// One entry of the frame's Types table: the fully qualified name of a concrete [SerializeReference] +// type, in the form GetRefTypeTypeTree matches against. +[StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] +public struct RegistryFrameType +{ + [MarshalAs(UnmanagedType.ByValTStr, SizeConst = 256)] + public string ClassName; + [MarshalAs(UnmanagedType.ByValTStr, SizeConst = 256)] + public string NamespaceName; + [MarshalAs(UnmanagedType.ByValTStr, SizeConst = 256)] + public string AssemblyName; +} + +[StructLayout(LayoutKind.Sequential)] +public struct RegistryFrameRecord +{ + public long Rid; + public int TypeIndex; // index into the Types table, or -1 for a null entry, which carries no blob + public uint ByteSize; + public ulong BlobOffset; +} + public static class DllWrapper { [DllImport("UnityFileSystemApi", @@ -318,6 +357,44 @@ public static extern ReturnCode GetTypeTreeNodeInfo(TypeTreeHandle handle, int n [MarshalAs(UnmanagedType.U4)] out TypeTreeMetaFlags metaFlags, out int firstChildNode, out int nextNode); + // As GetTypeTreeNodeInfo, but for a node inside a shared subtree. A null subtree means the type + // tree's own nodes, so this call alone covers both sides. offset is unset inside a subtree. + [DllImport("UnityFileSystemApi", + CallingConvention = CallingConvention.Cdecl, + EntryPoint = "UFS_GetTypeTreeSubtreeNodeInfo")] + public static extern ReturnCode GetTypeTreeSubtreeNodeInfo(TypeTreeHandle handle, IntPtr subtree, int node, + StringBuilder type, int typeLen, StringBuilder name, int nameLen, out int offset, out int size, + [MarshalAs(UnmanagedType.U4)] out TypeTreeFlags flags, [MarshalAs(UnmanagedType.U4)] out TypeTreeMetaFlags metaFlags, + out int firstChildNode, out int nextNode); + + // Resolves an IsSharedSubtreeRef node. The reference stands in for the subtree's root, so + // firstChildNode is that root's first child; 0 means the compound has no fields. + [DllImport("UnityFileSystemApi", + CallingConvention = CallingConvention.Cdecl, + EntryPoint = "UFS_GetTypeTreeRefSubtree")] + public static extern ReturnCode GetTypeTreeRefSubtree(TypeTreeHandle handle, IntPtr subtree, int node, + out IntPtr refSubtree, out int firstChildNode); + + // The registry frame is object data rather than type information, so these take the frame's bytes + // and a bound on them. The frame's own length field is untrusted, so size must be what the caller + // actually holds. + [DllImport("UnityFileSystemApi", + CallingConvention = CallingConvention.Cdecl, + EntryPoint = "UFS_GetRegistryFrameInfo")] + public static extern ReturnCode GetRegistryFrameInfo(IntPtr data, ulong size, out RegistryFrameInfo info); + + [DllImport("UnityFileSystemApi", + CallingConvention = CallingConvention.Cdecl, + EntryPoint = "UFS_GetRegistryFrameTypes")] + public static extern ReturnCode GetRegistryFrameTypes(IntPtr data, ulong size, + [Out] RegistryFrameType[] types, int len); + + [DllImport("UnityFileSystemApi", + CallingConvention = CallingConvention.Cdecl, + EntryPoint = "UFS_GetRegistryFrameRecords")] + public static extern ReturnCode GetRegistryFrameRecords(IntPtr data, ulong size, + [Out] RegistryFrameRecord[] records, int len); + [DllImport("UnityFileSystemApi", CallingConvention = CallingConvention.Cdecl, EntryPoint = "UFS_GetDllVersion")] diff --git a/UnityFileSystem/ManagedReferenceRegistry.cs b/UnityFileSystem/ManagedReferenceRegistry.cs new file mode 100644 index 0000000..0efdb30 --- /dev/null +++ b/UnityFileSystem/ManagedReferenceRegistry.cs @@ -0,0 +1,145 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Runtime.InteropServices; + +namespace UnityDataTools.FileSystem; + +// One [SerializeReference] instance owned by a serialized object. The fields that point at it store +// only its Rid, so a shared instance appears once here and is referred to from several places. +public sealed class ManagedReferenceEntry +{ + public long Rid { get; init; } + + // A null reference: no type, no data. Only the registry frame format can express one. + public bool IsNull { get; init; } + + // The instance's concrete type, which is what lays out its data. Empty when IsNull. + public string ClassName { get; init; } = ""; + public string Namespace { get; init; } = ""; + public string AssemblyName { get; init; } = ""; + + // Absolute position of the instance's data in the file. Zero when IsNull. + public long DataOffset { get; init; } +} + +// The [SerializeReference] instances owned by one serialized object, however the file stores them. +// +// From SerializedFile version 25 they live in a self-delimiting frame of raw bytes that no TypeTree +// node describes, sitting immediately before the field flagged HasSerializedRefs. Earlier files +// describe the registry with nodes, and callers read those through the node walk into these same +// entries. +// +// The frame's tables are not parsed here: UFS_GetRegistryFrame* wraps the engine's own parser, so +// this calls the one implementation that exists rather than becoming a second one. Only the 8-byte +// header is read directly, in GetFrameSize. +public sealed class ManagedReferenceRegistry +{ + // The only frame layout the native parser accepts. Unrelated to the serialized file version. + public const int FrameVersion = 3; + + // Frame header: [int32 version][uint32 byteLength], where byteLength counts to the frame's end. + const int FrameHeaderSize = 8; + + public int Version { get; private init; } + + // Bytes the frame occupies, counted from its first byte. Add to the frame's position to reach + // the data of the field it precedes. + public long FrameSize { get; private init; } + + // Absolute position of the first data blob, i.e. one past the frame's header and tables. + public long BlobsOffset { get; private init; } + + public IReadOnlyList Entries { get; private init; } + + // Builds a registry from entries a caller read some other way: versions 1 and 2 are described by + // TypeTree nodes, so the node walk produces the entries and they are presented the same way. + internal static ManagedReferenceRegistry FromEntries(int version, IReadOnlyList entries) + { + return new ManagedReferenceRegistry { Version = version, Entries = entries }; + } + + // Bytes the frame at `offset` occupies, read from its header alone. This is how a walker that + // only needs to step over the frame avoids parsing its tables. + // + // `availableBytes` is what the caller actually holds from `offset`, normally to the end of the + // object. The frame's own length field is untrusted and must never be used as the bound. + public static long GetFrameSize(UnityFileReader reader, long offset, long availableBytes) + { + if (availableBytes < FrameHeaderSize) + throw new InvalidDataException($"No room for a [SerializeReference] registry frame at offset {offset}"); + + var version = reader.ReadInt32(offset); + if (version != FrameVersion) + throw new InvalidDataException($"Unsupported [SerializeReference] registry version {version} at offset {offset}"); + + var frameSize = FrameHeaderSize + (long)reader.ReadUInt32(offset + 4); + if (frameSize > availableBytes) + throw new InvalidDataException($"[SerializeReference] registry frame at offset {offset} runs past the end of the object"); + + return frameSize; + } + + // Reads the frame at `offset`, including its tables. `availableBytes` is bounded as it is for + // GetFrameSize. Throws when there is no readable frame, which callers reach only where a node + // flagged HasSerializedRefs says one is present. + public static ManagedReferenceRegistry ReadFrame(UnityFileReader reader, long offset, long availableBytes) + { + var frameSize = GetFrameSize(reader, offset, availableBytes); + var bytes = new byte[frameSize]; + reader.ReadRange(offset, (int)frameSize, bytes); + + var pinned = GCHandle.Alloc(bytes, GCHandleType.Pinned); + try + { + var data = pinned.AddrOfPinnedObject(); + var size = (ulong)frameSize; + + var r = DllWrapper.GetRegistryFrameInfo(data, size, out var info); + if (r == ReturnCode.FileFormatError) + throw new InvalidDataException($"Malformed [SerializeReference] registry frame at offset {offset}"); + UnityFileSystem.HandleErrors(r); + + var types = new RegistryFrameType[info.TypeCount]; + if (info.TypeCount > 0) + UnityFileSystem.HandleErrors(DllWrapper.GetRegistryFrameTypes(data, size, types, types.Length)); + + var records = new RegistryFrameRecord[info.RecordCount]; + if (info.RecordCount > 0) + UnityFileSystem.HandleErrors(DllWrapper.GetRegistryFrameRecords(data, size, records, records.Length)); + + var entries = new List(records.Length); + foreach (var record in records) + { + if (record.TypeIndex < 0) + { + entries.Add(new ManagedReferenceEntry { Rid = record.Rid, IsNull = true }); + continue; + } + + var type = types[record.TypeIndex]; + entries.Add(new ManagedReferenceEntry + { + Rid = record.Rid, + ClassName = type.ClassName, + Namespace = type.NamespaceName, + AssemblyName = type.AssemblyName, + // Frame offsets are relative to its first byte. + DataOffset = offset + (long)record.BlobOffset, + }); + } + + return new ManagedReferenceRegistry + { + Version = info.Version, + FrameSize = (long)info.FrameSize, + BlobsOffset = offset + (long)info.BlobsOffset, + Entries = entries, + }; + } + finally + { + pinned.Free(); + } + } +} diff --git a/UnityFileSystem/SerializedFileOpenException.cs b/UnityFileSystem/SerializedFileOpenException.cs index 1dbc706..e19dcfc 100644 --- a/UnityFileSystem/SerializedFileOpenException.cs +++ b/UnityFileSystem/SerializedFileOpenException.cs @@ -25,4 +25,14 @@ public SerializedFileOpenException(string filePath, bool missingTypeTrees = fals FilePath = filePath; MissingTypeTrees = missingTypeTrees; } + + /// + /// Carries a specific reason instead of the generic one, for the cases the caller diagnosed + /// itself rather than learning from a failed open. + /// + public SerializedFileOpenException(string filePath, string reason) + : base(reason) + { + FilePath = filePath; + } } diff --git a/UnityFileSystem/TypeTreeNode.cs b/UnityFileSystem/TypeTreeNode.cs index 9a3ab49..536c7b3 100644 --- a/UnityFileSystem/TypeTreeNode.cs +++ b/UnityFileSystem/TypeTreeNode.cs @@ -11,6 +11,10 @@ public class TypeTreeNode int m_FirstChildNodeIndex; int m_NextNodeIndex; TypeTreeHandle m_Handle; + // The shared subtree this node lives in, or IntPtr.Zero for the type tree's own nodes. + // Only meaningful together with m_Handle, which owns it. + IntPtr m_Subtree; + int m_NodeIndex; Lazy> m_Children; Lazy m_CSharpType; Lazy m_hasConstantSize; @@ -30,11 +34,14 @@ public class TypeTreeNode // Child nodes container. public List Children => m_Children.Value; - // True if the field has no child. - public bool IsLeaf => m_FirstChildNodeIndex == 0; + // True if the field has no child. A shared subtree reference has to resolve its subtree to + // answer, since its children live there rather than at m_FirstChildNodeIndex. + public bool IsLeaf => IsSharedSubtreeRef ? Children.Count == 0 : m_FirstChildNodeIndex == 0; // True if the field is a basic type. (int, float, char, etc.) - public bool IsBasicType => IsLeaf && Size > 0; + // The IsSharedSubtreeRef test covers the one case IsLeaf cannot: a shared compound with no + // fields is a leaf with a byte size, which is indistinguishable from a primitive of that width. + public bool IsBasicType => !IsSharedSubtreeRef && IsLeaf && Size > 0; // True if the field is an array. public bool IsArray => ((int)Flags & (int)TypeTreeFlags.IsArray) != 0; @@ -42,6 +49,15 @@ public class TypeTreeNode // True if the field is a ManagedReferenceRegistry public bool IsManagedReferenceRegistry => ((int)Flags & (int)TypeTreeFlags.IsManagedReferenceRegistry) != 0; + // True if a version 3 [SerializeReference] registry sits in the data immediately before this + // field. The flag rides the declaring class's first field, reference or not, and is meaningful + // only when the tree is read as an object root. See ManagedReferenceRegistry. + public bool HasSerializedRefs => ((int)Flags & (int)TypeTreeFlags.HasSerializedRefs) != 0; + + // True if the node stands in for a compound the file stores once and shares between types + // (SerializedFile version 26 and later). Its children come from that shared subtree. + public bool IsSharedSubtreeRef => ((int)Flags & (int)TypeTreeFlags.IsSharedSubtreeRef) != 0; + // C# type corresponding to the node type public Type CSharpType => m_CSharpType.Value; @@ -79,10 +95,17 @@ static StringBuilder NodeNameBuilder } internal TypeTreeNode(TypeTreeHandle typeTreeHandle, int nodeIndex) + : this(typeTreeHandle, IntPtr.Zero, nodeIndex) + { + } + + internal TypeTreeNode(TypeTreeHandle typeTreeHandle, IntPtr subtree, int nodeIndex) { m_Handle = typeTreeHandle; + m_Subtree = subtree; + m_NodeIndex = nodeIndex; - var r = DllWrapper.GetTypeTreeNodeInfo(m_Handle, nodeIndex, NodeTypeBuilder, NodeTypeBuilder.Capacity, NodeNameBuilder, NodeNameBuilder.Capacity, out Offset, out Size, out Flags, out MetaFlags, out m_FirstChildNodeIndex, out m_NextNodeIndex); + var r = DllWrapper.GetTypeTreeSubtreeNodeInfo(m_Handle, m_Subtree, nodeIndex, NodeTypeBuilder, NodeTypeBuilder.Capacity, NodeNameBuilder, NodeNameBuilder.Capacity, out Offset, out Size, out Flags, out MetaFlags, out m_FirstChildNodeIndex, out m_NextNodeIndex); UnityFileSystem.HandleErrors(r); Type = NodeTypeBuilder.ToString(); @@ -96,11 +119,20 @@ internal TypeTreeNode(TypeTreeHandle typeTreeHandle, int nodeIndex) internal List GetChildren() { var children = new List(); + var subtree = m_Subtree; var current = m_FirstChildNodeIndex; + if (IsSharedSubtreeRef) + { + // The reference stands in for the shared subtree's root, so the nodes that replace it + // are that root's children, numbered within the subtree it returns. + var r = DllWrapper.GetTypeTreeRefSubtree(m_Handle, m_Subtree, m_NodeIndex, out subtree, out current); + UnityFileSystem.HandleErrors(r); + } + while (current != 0) { - var child = new TypeTreeNode(m_Handle, current); + var child = new TypeTreeNode(m_Handle, subtree, current); children.Add(child); current = child.m_NextNodeIndex; } @@ -169,6 +201,13 @@ Type GetCSharpType() default: { + // A shared subtree reference carries a real byte size, so the size-based guesses + // below would read a compound as the primitive of that width. + if (IsSharedSubtreeRef) + { + return typeof(object); + } + if (Size == 8) { return typeof(long); diff --git a/UnityFileSystem/TypeTreeReaders/RandomAccessReader.cs b/UnityFileSystem/TypeTreeReaders/RandomAccessReader.cs index 4537a84..8c0fd9a 100644 --- a/UnityFileSystem/TypeTreeReaders/RandomAccessReader.cs +++ b/UnityFileSystem/TypeTreeReaders/RandomAccessReader.cs @@ -27,6 +27,15 @@ public class RandomAccessReader : IEnumerable Dictionary m_ChildrenCacheObject; List m_ChildrenCacheArray; private TypeTreeNode m_TypeTreeNode; + // A registry frame is honoured in root context only; a referenced instance's data has none. + bool m_IsRoot; + ManagedReferenceRegistry m_Registry; + // Where this object's registry frame starts, once the field walk has passed it. -1 until then, + // and for the versions that describe the registry with nodes instead. + long m_RegistryFrameOffset = -1; + // End of the object being read, when the caller knew it. The frame's own length field is + // untrusted, so it must be bounded by something the caller actually holds. + long m_ObjectEnd; public int Size => m_Size.Value; public long Offset { get; } @@ -36,9 +45,21 @@ public class RandomAccessReader : IEnumerable public bool IsArrayOfBasicTypes => m_TypeTreeNode.IsArray && m_TypeTreeNode.Children[1].IsBasicType; public bool IsArray => m_TypeTreeNode.IsArray; - public RandomAccessReader(SerializedFile serializedFile, TypeTreeNode node, UnityFileReader reader, long offset, bool isReferencedObject = false) + // objectSize is the size of the object at `offset`, from its ObjectInfo. It bounds the + // [SerializeReference] registry frame read, whose own length field cannot be trusted; pass it + // whenever it is known. Without it the bound is the rest of the file. + public RandomAccessReader(SerializedFile serializedFile, TypeTreeNode node, UnityFileReader reader, long offset, + bool isReferencedObject = false, long objectSize = 0) + : this(serializedFile, node, reader, offset, isReferencedObject, isRoot: true, objectSize) + { + } + + RandomAccessReader(SerializedFile serializedFile, TypeTreeNode node, UnityFileReader reader, long offset, + bool isReferencedObject, bool isRoot, long objectSize = 0) { m_SerializedFile = serializedFile; + m_IsRoot = isRoot; + m_ObjectEnd = objectSize > 0 ? offset + objectSize : 0; // Special case for vector and map objects, they always have a single Array child so we skip it. if (node.Type == "vector" || node.Type == "map" || node.Type == "staticvector") @@ -63,11 +84,13 @@ public RandomAccessReader(SerializedFile serializedFile, TypeTreeNode node, Unit // created and don't match the TypeTree. if (m_TypeTreeNode.IsManagedReferenceRegistry) { - var versionReader = new RandomAccessReader(m_SerializedFile, node.Children[0], reader, offset); + var versionReader = new RandomAccessReader(m_SerializedFile, node.Children[0], reader, offset, isReferencedObject: false, isRoot: false); m_ChildrenCacheObject["version"] = versionReader; int version = versionReader.GetValue(); long curOffset = versionReader.Offset + versionReader.Size; + var entries = new List(); + if (version == 1) { // Second child is the ReferencedObject. @@ -78,7 +101,7 @@ public RandomAccessReader(SerializedFile serializedFile, TypeTreeNode node, Unit do { // Create the referenced object reader. - var refObjReader = new RandomAccessReader(m_SerializedFile, refObjNode, reader, curOffset, true); + var refObjReader = new RandomAccessReader(m_SerializedFile, refObjNode, reader, curOffset, isReferencedObject: true, isRoot: false); // A referenced object with null data means that we reached the end of the referenced objects. if (refObjReader["data"] == null) @@ -87,7 +110,8 @@ public RandomAccessReader(SerializedFile serializedFile, TypeTreeNode node, Unit } // Add the reader to cache. - m_ChildrenCacheObject[$"rid({i++})"] = refObjReader; + m_ChildrenCacheObject[$"rid({i})"] = refObjReader; + entries.Add(MakeEntry(refObjReader, i++)); curOffset += refObjReader.Size; } while (true); } @@ -110,8 +134,10 @@ public RandomAccessReader(SerializedFile serializedFile, TypeTreeNode node, Unit for (int i = 0; i < arraySize; ++i) { // Create and cache the referenced object. - var refObjReader = new RandomAccessReader(m_SerializedFile, refObjNode, reader, curOffset, true); - m_ChildrenCacheObject[$"rid({refObjReader["rid"].GetValue()})"] = refObjReader; + var refObjReader = new RandomAccessReader(m_SerializedFile, refObjNode, reader, curOffset, isReferencedObject: true, isRoot: false); + var rid = refObjReader["rid"].GetValue(); + m_ChildrenCacheObject[$"rid({rid})"] = refObjReader; + entries.Add(MakeEntry(refObjReader, rid)); curOffset += refObjReader.Size; } } @@ -119,6 +145,8 @@ public RandomAccessReader(SerializedFile serializedFile, TypeTreeNode node, Unit { throw new Exception($"Unsupported ManagedReferenceRegistry version {version}"); } + + m_Registry = ManagedReferenceRegistry.FromEntries(version, entries); } else if (isReferencedObject) { @@ -152,13 +180,71 @@ public RandomAccessReader(SerializedFile serializedFile, TypeTreeNode node, Unit // Manually create and cache a reader for the referenced type data, using its own TypeTree. var refTypeDataReader = new RandomAccessReader(m_SerializedFile, refTypeRoot, reader, - referencedManagedType.Offset + referencedManagedType.Size); + referencedManagedType.Offset + referencedManagedType.Size, isReferencedObject: false, isRoot: false); m_ChildrenCacheObject["data"] = refTypeDataReader; } } } } + // The [SerializeReference] instances this object owns, or null when it has none. Reading it + // walks the object's fields, since that is what locates the registry in either layout. + public ManagedReferenceRegistry Registry + { + get + { + if (m_Registry != null) + return m_Registry; + + if (!m_IsRoot || !IsObject) + return null; + + foreach (var child in m_TypeTreeNode.Children) + { + // Versions 1 and 2 describe the registry with a node of its own. + if (child.IsManagedReferenceRegistry) + return GetChild(child.Name).m_Registry; + + // Version 3 has no node; reading the field the frame precedes walks past it, which + // is what records where it starts. + if (child.HasSerializedRefs) + { + GetChild(child.Name); + m_Registry = ManagedReferenceRegistry.ReadFrame(m_Reader, m_RegistryFrameOffset, + RemainingBytes(m_RegistryFrameOffset)); + return m_Registry; + } + } + + return null; + } + } + + // What the caller holds from `offset`: the object's own bytes when its size was given, and the + // rest of the file otherwise. + long RemainingBytes(long offset) => (m_ObjectEnd > 0 ? m_ObjectEnd : m_Reader.Length) - offset; + + // Maps one version 1 or 2 registry entry, already read through its TypeTree nodes, onto the + // shared entry shape. + static ManagedReferenceEntry MakeEntry(RandomAccessReader referencedObject, long rid) + { + var data = referencedObject["data"]; + + if (data == null) + return new ManagedReferenceEntry { Rid = rid, IsNull = true }; + + var type = referencedObject["type"]; + + return new ManagedReferenceEntry + { + Rid = rid, + ClassName = type["class"].GetValue(), + Namespace = type["ns"].GetValue(), + AssemblyName = type["asm"].GetValue(), + DataOffset = data.Offset, + }; + } + public bool HasChild(string name) { // Special case for ManagedReferenceRegistry. The children are in cache and do not match the TypeTreeNode. @@ -210,6 +296,12 @@ int GetSize() { size = m_Reader.ReadInt32(Offset) + 4; } + else if (m_TypeTreeNode.Children.Count == 0) + { + // A compound with no fields, which a [Serializable] class with nothing serialized + // reaches. It occupies whatever its own node says, normally nothing. + size = Math.Max(m_TypeTreeNode.Size, 0); + } else { var lastChild = GetChild(m_TypeTreeNode.Children.Last().Name); @@ -258,7 +350,15 @@ RandomAccessReader GetChild(string name) { var child = m_TypeTreeNode.Children[i]; - nodeReader = new RandomAccessReader(m_SerializedFile, child, m_Reader, offset); + // A version 3 registry sits in the data before this field, so the field starts past + // it. Only its size is needed to get there; Registry reads the tables on demand. + if (m_IsRoot && child.HasSerializedRefs) + { + m_RegistryFrameOffset = offset; + offset += ManagedReferenceRegistry.GetFrameSize(m_Reader, offset, RemainingBytes(offset)); + } + + nodeReader = new RandomAccessReader(m_SerializedFile, child, m_Reader, offset, isReferencedObject: false, isRoot: false); m_ChildrenCacheObject.Add(child.Name, nodeReader); m_LastCachedChild = nodeReader; @@ -314,7 +414,7 @@ RandomAccessReader GetArrayElement(int index) for (int i = m_ChildrenCacheArray.Count; i < arraySize; ++i) { - nodeReader = new RandomAccessReader(m_SerializedFile, dataNode, m_Reader, offset); + nodeReader = new RandomAccessReader(m_SerializedFile, dataNode, m_Reader, offset, isReferencedObject: false, isRoot: false); m_ChildrenCacheArray.Add(nodeReader); m_LastCachedChild = nodeReader; diff --git a/UnityFileSystem/UnityFileReader.cs b/UnityFileSystem/UnityFileReader.cs index 73221be..df47f61 100644 --- a/UnityFileSystem/UnityFileReader.cs +++ b/UnityFileSystem/UnityFileReader.cs @@ -51,6 +51,20 @@ public void ReadArray(long fileOffset, int size, Array dest) Buffer.BlockCopy(m_Buffer, offset, dest, 0, size); } + // Reads a range that may be larger than the internal buffer, in buffer-sized chunks. + public void ReadRange(long fileOffset, int size, byte[] dest) + { + var written = 0; + + while (written < size) + { + var chunk = Math.Min(m_Buffer.Length, size - written); + var offset = GetBufferOffset(fileOffset + written, chunk); + Buffer.BlockCopy(m_Buffer, offset, dest, written, chunk); + written += chunk; + } + } + public string ReadString(long fileOffset, int size) { var offset = GetBufferOffset(fileOffset, size); diff --git a/UnityFileSystem/UnityFileSystem.cs b/UnityFileSystem/UnityFileSystem.cs index a623860..839538b 100644 --- a/UnityFileSystem/UnityFileSystem.cs +++ b/UnityFileSystem/UnityFileSystem.cs @@ -7,6 +7,11 @@ namespace UnityDataTools.FileSystem; // This is the main entry point. Provides methods to mount archives and open files. public static class UnityFileSystem { + // Serialized file version 26 (Unity 6.7) needs the shared subtree and registry frame entry + // points, which arrived together in this library version. Older files are read through the + // same library, so there is no reason to keep a fallback for an earlier one. + public const int RequiredDllVersion = 2; + public static void Init() { // Initialize the native library. @@ -16,6 +21,14 @@ public static void Init() { HandleErrors(r); } + + var dllVersion = GetDllVersion(); + if (dllVersion < RequiredDllVersion) + { + throw new NotSupportedException( + $"UnityFileSystemApi version {dllVersion} is too old; version {RequiredDllVersion} or newer is required. " + + "Replace the UnityFileSystemApi library shipped beside UnityDataTool with one from Unity 6.7 or newer."); + } } public static void Cleanup() @@ -121,6 +134,14 @@ internal static void HandleErrors(ReturnCode returnCode, string filename = "") case ReturnCode.TypeNotFound: throw new ArgumentException("Type not found."); + + case ReturnCode.HigherTypeTreeVersion: + throw new NotSupportedException($"A TypeTree in {filename} was written by a newer version of Unity."); + + // Every node is read through the subtree API, so the native side has no reason to ask + // for it. Reaching this means a walk was added that bypasses TypeTreeNode. + case ReturnCode.RequiresSubtreeApi: + throw new InvalidOperationException("Shared subtree reference read through the non-subtree API."); } } }