├── Hashing
├── IBundleHashingMethod.cs
└── MD5Hasher.cs
├── LegacyBundleHashes.asmdef
├── LegacyBundleHashing.cs
├── README.md
├── Reporting
└── HashReporter.cs
└── Unity Tool Processors
├── AssetBundleUnpacker.cs
└── Binary2TextProcessor.cs
/Hashing/IBundleHashingMethod.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Collections;
3 | using System.Collections.Generic;
4 | using System.IO;
5 | using UnityEngine;
6 |
7 | namespace BundleHashing
8 | {
9 | public interface IBundleHashingMethod : IDisposable
10 | {
11 | ///
12 | ///
13 | ///
14 | ///
15 | void Feed( FileInfo file );
16 |
17 | ///
18 | ///
19 | ///
20 | ///
21 | string Complete();
22 | }
23 | }
24 |
--------------------------------------------------------------------------------
/Hashing/MD5Hasher.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.IO;
3 | using System.Security.Cryptography;
4 | using System.Text;
5 | using UnityEditor;
6 | using UnityEngine;
7 |
8 | namespace BundleHashing
9 | {
10 | public class MD5Hasher : IBundleHashingMethod
11 | {
12 | private MD5 m_MD5;
13 | private MD5 Md5
14 | {
15 | get
16 | {
17 | if( m_MD5 == null )
18 | m_MD5 = MD5.Create();
19 | return m_MD5;
20 | }
21 | }
22 |
23 | ///
24 | ///
25 | ///
26 | ///
27 | public void Feed( FileInfo file )
28 | {
29 | using( var stream = file.OpenRead( ) )
30 | {
31 | byte[] buffer = new byte[4096];
32 | int read = 0;
33 | while ((read = stream.Read(buffer, 0, buffer.Length)) > 0)
34 | {
35 | Md5.TransformBlock(buffer, 0, read, buffer, 0);
36 | }
37 | }
38 | }
39 |
40 | ///
41 | ///
42 | ///
43 | ///
44 | public string Complete()
45 | {
46 | byte[] data = new byte[0];
47 | Md5.TransformFinalBlock(data, 0, 0);
48 | data = Md5.Hash;
49 | StringBuilder final = new StringBuilder(32);
50 | for (int i = 0; i < data.Length; i++)
51 | final.Append(data[i].ToString("x2"));
52 |
53 | m_MD5?.Initialize();
54 | return final.ToString();
55 | }
56 |
57 | ///
58 | ///
59 | ///
60 | public void Dispose()
61 | {
62 | m_MD5?.Dispose();
63 | }
64 | }
65 | }
--------------------------------------------------------------------------------
/LegacyBundleHashes.asmdef:
--------------------------------------------------------------------------------
1 | {
2 | "name": "LegacyBundleHashes",
3 | "references": [],
4 | "optionalUnityReferences": [],
5 | "includePlatforms": [
6 | "Editor"
7 | ],
8 | "excludePlatforms": [],
9 | "allowUnsafeCode": false,
10 | "overrideReferences": false,
11 | "precompiledReferences": [],
12 | "autoReferenced": true,
13 | "defineConstraints": []
14 | }
--------------------------------------------------------------------------------
/LegacyBundleHashing.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Collections;
3 | using System.Collections.Generic;
4 | using System.IO;
5 | using System.Security.AccessControl;
6 | using UnityEngine;
7 |
8 | namespace BundleHashing
9 | {
10 | ///
11 | ///
12 | ///
13 | public class LegacyBundleHashing
14 | {
15 | private bool m_IsDone = false;
16 | ///
17 | ///
18 | ///
19 | public bool IsDone
20 | {
21 | get { return m_IsDone; }
22 | }
23 |
24 | ///
25 | ///
26 | ///
27 | public Action OnCompleted;
28 |
29 | ///
30 | ///
31 | ///
32 | ///
33 | ///
34 | ///
35 | ///
36 | ///
37 | public static LegacyBundleHashing GenerateAssetBundleHashes( string buildPath, AssetBundleManifest manifest, string outputPath, IBundleHashingMethod hashingMethod = null )
38 | {
39 | string[] bundles = manifest.GetAllAssetBundles();
40 | string[] bundlesPaths = new string[bundles.Length];
41 | for( int i = 0; i < bundles.Length; ++i )
42 | bundlesPaths[i] = Path.Combine( buildPath, bundles[i] );
43 |
44 | return GenerateAssetBundleHashes( bundlesPaths, outputPath, hashingMethod );
45 | }
46 |
47 | ///
48 | ///
49 | ///
50 | ///
51 | ///
52 | ///
53 | ///
54 | public static LegacyBundleHashing GenerateAssetBundleHashes( IList bundlePaths, string outputPath, IBundleHashingMethod hashingMethod = null )
55 | {
56 | List files = new List(bundlePaths.Count);
57 | foreach( string path in bundlePaths )
58 | {
59 | if( File.Exists( path ) )
60 | files.Add( new FileInfo( path ) );
61 | else
62 | Debug.LogError( "Could not find AssetBundle at " + path );
63 | }
64 | return GenerateAssetBundleHashes( files, outputPath, hashingMethod );
65 | }
66 |
67 | ///
68 | ///
69 | ///
70 | ///
71 | ///
72 | ///
73 | ///
74 | public static LegacyBundleHashing GenerateAssetBundleHashes( IList bundleFiles, string outputPath, IBundleHashingMethod hashingMethod = null )
75 | {
76 | return new LegacyBundleHashing( bundleFiles, outputPath, hashingMethod );
77 | }
78 |
79 | private IList m_AssetBundles;
80 | private AssetBundleUnpacker m_Unpacker;
81 | private Binary2TextProcessor m_2texter;
82 | private IBundleHashingMethod m_Hasher;
83 | private HashReporter m_Reporter;
84 |
85 | private LegacyBundleHashing( IList bundlePaths, string outputPath, IBundleHashingMethod hashingMethod = null )
86 | {
87 | m_Hasher = hashingMethod != null ? hashingMethod : new MD5Hasher();
88 | m_Reporter = new HashReporter( outputPath );
89 | m_AssetBundles = bundlePaths;
90 | m_Unpacker = new AssetBundleUnpacker( bundlePaths );
91 | m_Unpacker.OnCompleted = OnUnpacked;
92 | m_Unpacker.Start();
93 | }
94 |
95 | private void OnUnpacked()
96 | {
97 | // Now we need to convert any .sharedresource files or no extention files to text
98 | // this will eliminate any unwanted header information being included in the hashing
99 |
100 | List dirs = m_Unpacker.UnpackedDirectories;
101 | List filesToConvert = new List(dirs.Count * 2);
102 | foreach( DirectoryInfo dir in dirs )
103 | GetFilesForConverting( dir, filesToConvert );
104 |
105 | m_2texter = new Binary2TextProcessor( filesToConvert );
106 | m_2texter.OnCompleted = CalculateHashes;
107 | m_2texter.Start();
108 | }
109 |
110 | private void CalculateHashes()
111 | {
112 | List unpackedDirectories = m_Unpacker.UnpackedDirectories;
113 | if( m_AssetBundles.Count != unpackedDirectories.Count )
114 | {
115 | Debug.LogError( "Unknown issue, extracted output != assetBundle input" );
116 | return;
117 | }
118 |
119 | for( int i = 0; i < unpackedDirectories.Count; ++i )
120 | {
121 | IEnumerable files = unpackedDirectories[i].EnumerateFiles("*", SearchOption.AllDirectories);
122 | foreach( FileInfo file in files )
123 | {
124 | if( file.Extension == ".resS" || file.Extension == ".resource" || file.Extension == ".txt" )
125 | {
126 | m_Hasher.Feed( file );
127 | }
128 | }
129 |
130 | string hash = m_Hasher.Complete();
131 | Debug.Log( "hash for " + m_AssetBundles[i].Name + " is " + hash );
132 | m_Reporter.AddAssetBundle( m_AssetBundles[i], hash );
133 | }
134 |
135 | m_Reporter.Write();
136 | Cleanup();
137 | OnCompleted?.Invoke();
138 | }
139 |
140 | private void Cleanup()
141 | {
142 | List unpackedDirectories = m_Unpacker.UnpackedDirectories;
143 | foreach( DirectoryInfo unpackedDirectory in unpackedDirectories )
144 | {
145 | if( unpackedDirectory.Exists )
146 | unpackedDirectory.Delete(true);
147 | }
148 |
149 | m_Hasher.Dispose(); // Only if not user provided?
150 | m_AssetBundles = null;
151 | m_Unpacker = null;
152 | m_2texter = null;
153 | }
154 |
155 | private static void GetFilesForConverting( DirectoryInfo d, List files )
156 | {
157 | foreach( FileInfo file in d.EnumerateFiles() )
158 | {
159 | if( File.Exists( file.FullName ) == false )
160 | {
161 | Debug.LogError( "How did this happen?" );
162 | continue;
163 | }
164 | if( file.Extension == ".sharedAssets" || file.Extension == "" )
165 | files.Add( file );
166 | }
167 |
168 | // expect this to be never needed.. \(o.o)/
169 | foreach( DirectoryInfo directory in d.EnumerateDirectories() )
170 | {
171 | if( directory.FullName != d.FullName )
172 | GetFilesForConverting( directory, files );
173 | }
174 | }
175 | }
176 | }
177 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # AssetBundle-ContentHasher
2 | This tool can be integrated into a build pipeline in order to generate more reliable hashes for AssetBundles using Unity's built in pipeline.
3 |
4 | When building with the built in buildPipeline (Not with ScriptableBuildPipeline) https://docs.unity3d.com/2019.2/Documentation/Manual/AssetBundles-Building.html. The AssetBundle's are accompanied by .manifest files. The assetHash in these files are the mechanism Unity uses to determine if an AssetBundle needs to be rebuilt. It uses the hashes that are used to determine if an Asset needs to be reimported. Doing so on the Assets set to the AssetBundle to determine this assetHash for the AssetBundle.
5 | This hash is calculated by the binary content of the file itself (In the Assets folder, and not the built asset). The platform and importer versions, any post processor versions affecting the Asset, and the .meta file binary.
6 |
7 | This assetHash has quite a few flaws, if used as a mechanism for identifying the contents of an AssetBundle. Such as: MonoBehaviour on a script is just a GUID and FileID to reference it in the Editor AssetDatabase (Allowing changes such as filename, namespace etc). So if you were to change something about the MonoScript that it uses to load the Script at runtime. e.g. The namespace. Then the AssetHash of the Scene will not change. This could lead to the AssetBundle not rebuilding or if you were to use the assetHash for the cacheHash which is a common issue. The the bundle for the Scene may be loading an old version, that attempts to load a Script with invalid information. Though these issues are a rare situation for many people, they are hard to debug when they occur; and often happen in live projects which is a big problem.
8 |
9 | Unity uses the assetHash for estimating if an AssetBundles need to be rebuilt, improving development iteration times, by not rebuilding AssetBundles during development. Due to the flaws however. You should always do a ForceRebuildAssetBundles build option when you need to make sure that the AssetBundles are correct for the project content. Such as beta or especially final release builds.
10 |
11 | The problem comes when this hash is used as an identifier for an AssetBundle during a live game. Such as using the assetHash as the cacheHash (version) when downloading using UnityWebRequest. Because on rare occasions an AssetBundle with content changes, could result in the same assetHash. This resulted in old AssetBundles being used from the cache instead of downloaded the updated ones.
12 |
13 | This is where this tool comes into play. Because it generates a hash from the built uncompressed content data contained within an AssetBundle. It will always change whenever the content changes.
14 |
15 | Why can I just not use the CRC of the AssetBundle file contained in the manifest?
16 | The CRC of an AssetBundle is calculated based upon the entire uncompressed bytes, and not just of the data. This includes the AssetBundle header information. This includes the Unity version that built the file, as a result of which. When building AssetBundles between different Editor versions the CRC will change. So this cannot be used as an identifier of if the content of the AssetBundle has been changed.
17 |
18 | # How to use
19 |
20 | Here is an example method for Building AssetBundles which will then calculate the hashes for the AssetBundles and write a report upon completion.
21 |
22 | ```c#
23 | [MenuItem("Assets/Build AssetBundles/With Hash Report")]
24 | public static void BuildAssetBundle()
25 | {
26 | string buildLocation = "Assets/StreamingAssets";
27 | if( ! Directory.Exists(buildLocation) )
28 | Directory.CreateDirectory(buildLocation);
29 |
30 | AssetBundleManifest m = BuildPipeline.BuildAssetBundles( buildLocation, BuildAssetBundleOptions.ForceRebuildAssetbundle, EditorUserBuildSettings.activeBuiltTarget );
31 | var hasher = BundleHashing.LegacyBundleHashing.GenerateAssetBundleHashes( buildLocation, manifest, Path.Combine(buildLocation, "AssetBundleDetails.json") );
32 | if( buildLocation.StartsWith( "Assets/" ) )
33 | hasher.OnCompleted = AssetDatabase.Refresh;
34 | }
35 | ```
36 |
--------------------------------------------------------------------------------
/Reporting/HashReporter.cs:
--------------------------------------------------------------------------------
1 | using System.Collections.Generic;
2 | using System.IO;
3 | using System.Text;
4 | using UnityEditor;
5 |
6 | namespace BundleHashing
7 | {
8 | ///
9 | ///
10 | ///
11 | public class HashReporter
12 | {
13 | private string m_OutputFilePath;
14 | private List> m_Data = new List>();
15 |
16 | ///
17 | ///
18 | ///
19 | ///
20 | public HashReporter( string outputFilePath )
21 | {
22 | m_OutputFilePath = outputFilePath;
23 | }
24 |
25 | ///
26 | ///
27 | ///
28 | ///
29 | ///
30 | public void AddAssetBundle( FileInfo bundle, string hash )
31 | {
32 | m_Data.Add( new KeyValuePair(bundle.FullName, hash) );
33 | }
34 |
35 | ///
36 | ///
37 | ///
38 | public void Write()
39 | {
40 | StringBuilder json = new StringBuilder("[\n");
41 |
42 | foreach( KeyValuePair pair in m_Data )
43 | {
44 | json.Append( "{ \"AssetBundle\": \"" );
45 | json.Append( pair.Key );
46 | json.Append( "\", \"Hash\": \"" );
47 | json.Append( pair.Value );
48 | json.Append( "\", \"CRC\": " );
49 | uint crc = 0;
50 | BuildPipeline.GetCRCForAssetBundle( pair.Key, out crc );
51 | json.Append( crc.ToString() );
52 | json.Append( " }\n" );
53 | }
54 | json.Append( "]" );
55 |
56 | File.WriteAllText( m_OutputFilePath, json.ToString() );
57 | }
58 | }
59 | }
--------------------------------------------------------------------------------
/Unity Tool Processors/AssetBundleUnpacker.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Collections.Generic;
3 | using System.Diagnostics;
4 | using System.IO;
5 | using UnityEditor;
6 | using Debug = UnityEngine.Debug;
7 |
8 | namespace BundleHashing
9 | {
10 | ///
11 | ///
12 | ///
13 | public class AssetBundleUnpacker
14 | {
15 | private static List m_Active = new List();
16 |
17 | private List m_AssetBundleFiles;
18 | private List m_UnpackedDirectories;
19 | ///
20 | ///
21 | ///
22 | public List UnpackedDirectories
23 | {
24 | get { return m_UnpackedDirectories; }
25 | }
26 |
27 | private Process m_Process = null;
28 | private bool m_HasStarted = false;
29 | private int m_FileIndex = 0;
30 |
31 | private bool m_IsDone = false;
32 | ///
33 | ///
34 | ///
35 | public bool IsDone
36 | {
37 | get { return m_IsDone; }
38 | }
39 |
40 | ///
41 | ///
42 | ///
43 | public Action OnCompleted;
44 |
45 | ///
46 | ///
47 | ///
48 | ///
49 | public AssetBundleUnpacker( IList assetBundleFilePaths )
50 | {
51 | m_AssetBundleFiles = new List( assetBundleFilePaths.Count );
52 | foreach( string filePath in assetBundleFilePaths )
53 | m_AssetBundleFiles.Add( new FileInfo( filePath ) );
54 | }
55 |
56 | ///
57 | ///
58 | ///
59 | ///
60 | public AssetBundleUnpacker( IList assetBundleFiles )
61 | {
62 | m_AssetBundleFiles = new List(assetBundleFiles);
63 | }
64 |
65 | ///
66 | ///
67 | ///
68 | public void Start()
69 | {
70 | if( m_HasStarted || m_IsDone )
71 | return;
72 |
73 | if( m_AssetBundleFiles.Count == 0 )
74 | m_IsDone = true;
75 | m_UnpackedDirectories = new List( m_AssetBundleFiles.Count );
76 | for( int i=0; i= 0; --i )
97 | m_Active[i].Update();
98 | }
99 |
100 | // May be able to run multiple processes, but I expect little gain
101 | private void Update()
102 | {
103 | if( m_Process == null )
104 | {
105 | if( m_FileIndex < m_AssetBundleFiles.Count )
106 | {
107 | UnpackAssetBundleFile( m_AssetBundleFiles[m_FileIndex] );
108 | }
109 | else
110 | {
111 | m_Active.Remove( this );
112 | if( m_Active.Count == 0 )
113 | {
114 | EditorApplication.update -= UpdateEditor;
115 | EditorUtility.ClearProgressBar();
116 | }
117 |
118 | m_IsDone = true;
119 | OnCompleted?.Invoke();
120 | }
121 | }
122 | }
123 |
124 | private void UnpackAssetBundleFile( FileInfo file )
125 | {
126 | EditorUtility.DisplayProgressBar( "Unpacking AssetBundle for hashing", file.FullName, (float) m_FileIndex / m_AssetBundleFiles.Count );
127 | m_Process = new Process
128 | {
129 | EnableRaisingEvents = true
130 | };
131 |
132 | m_Process.StartInfo.FileName = EditorApplication.applicationPath + "/Contents/Tools/WebExtract";
133 | m_Process.StartInfo.Arguments = "\"" + file.FullName + "\"";
134 | m_Process.StartInfo.UseShellExecute = false;
135 | m_Process.StartInfo.RedirectStandardOutput = true;
136 |
137 | m_Process.Exited += WebExtractProcess_Exited;
138 | m_Process.OutputDataReceived += WebExtractProcess_Output;
139 |
140 | try
141 | {
142 | m_Process.Start();
143 | m_Process.BeginOutputReadLine();
144 | }
145 | catch( Exception e )
146 | {
147 | Debug.LogError( "Failed to start process : " + e.Message );
148 | }
149 | }
150 |
151 | private void WebExtractProcess_Exited( object sender, EventArgs e )
152 | {
153 | m_Process.Close();
154 | m_Process.Dispose();
155 | m_Process = null;
156 | m_FileIndex++;
157 | }
158 |
159 | private void WebExtractProcess_Output( object sender, DataReceivedEventArgs e )
160 | {
161 | if( string.IsNullOrEmpty( e.Data ) )
162 | return;
163 |
164 | if( e.Data.Contains( "creating folder" ) )
165 | {
166 | var split = e.Data.Split( '\'' );
167 | if( Directory.Exists( split[1] ) == false )
168 | Debug.LogError( "Failed to find unpacked folder" );
169 | else
170 | m_UnpackedDirectories[m_FileIndex] = new DirectoryInfo( split[1] );
171 | }
172 | }
173 | }
174 | }
175 |
--------------------------------------------------------------------------------
/Unity Tool Processors/Binary2TextProcessor.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Collections.Generic;
3 | using System.Diagnostics;
4 | using System.IO;
5 | using UnityEditor;
6 | using Debug = UnityEngine.Debug;
7 |
8 | namespace BundleHashing
9 | {
10 | ///
11 | ///
12 | ///
13 | public class Binary2TextProcessor
14 | {
15 | private static List m_Active = new List();
16 |
17 | private List m_BinaryFiles;
18 | private List m_TextFiles;
19 | ///
20 | ///
21 | ///
22 | public List TextFiles
23 | {
24 | get { return m_TextFiles; }
25 | }
26 |
27 | private Process m_Process = null;
28 | private bool m_HasStarted = false;
29 | private int m_FileIndex = 0;
30 |
31 | private bool m_IsDone = false;
32 | ///
33 | ///
34 | ///
35 | public bool IsDone
36 | {
37 | get { return m_IsDone; }
38 | }
39 |
40 | ///
41 | ///
42 | ///
43 | public Action OnCompleted;
44 |
45 | ///
46 | ///
47 | ///
48 | ///
49 | public Binary2TextProcessor( IList assetBundleFilePaths )
50 | {
51 | m_BinaryFiles = new List( assetBundleFilePaths.Count );
52 | foreach( string filePath in assetBundleFilePaths )
53 | m_BinaryFiles.Add( new FileInfo( filePath ) );
54 | }
55 |
56 | ///
57 | ///
58 | ///
59 | ///
60 | public Binary2TextProcessor( IList assetBundleFiles )
61 | {
62 | m_BinaryFiles = new List(assetBundleFiles);
63 | }
64 |
65 | ///
66 | ///
67 | ///
68 | public void Start()
69 | {
70 | if( m_HasStarted || m_IsDone )
71 | return;
72 |
73 | if( m_BinaryFiles.Count == 0 )
74 | m_IsDone = true;
75 | m_TextFiles = new List( m_BinaryFiles.Count );
76 | for( int i=0; i= 0; --i )
97 | m_Active[i].Update();
98 | }
99 |
100 | // May be able to run multiple processes, but I expect little gain
101 | private void Update()
102 | {
103 | if( m_Process == null )
104 | {
105 | if( m_FileIndex < m_BinaryFiles.Count )
106 | {
107 | ConvertBinaryFileToText( m_BinaryFiles[m_FileIndex] );
108 | }
109 | else
110 | {
111 | m_Active.Remove( this );
112 | if( m_Active.Count == 0 )
113 | {
114 | EditorApplication.update -= UpdateEditor;
115 | EditorUtility.ClearProgressBar();
116 | }
117 |
118 | m_IsDone = true;
119 | OnCompleted?.Invoke();
120 | }
121 | }
122 | }
123 |
124 | private void ConvertBinaryFileToText( FileInfo file )
125 | {
126 | EditorUtility.DisplayProgressBar( "Converting to text for hashing", file.FullName, (float) m_FileIndex / m_BinaryFiles.Count );
127 | m_Process = new Process
128 | {
129 | EnableRaisingEvents = true
130 | };
131 |
132 | m_Process.StartInfo.FileName = EditorApplication.applicationPath + "/Contents/Tools/binary2text";
133 | m_Process.StartInfo.Arguments = "\"" + file.FullName + "\"";
134 | m_Process.StartInfo.UseShellExecute = false;
135 | m_Process.StartInfo.RedirectStandardOutput = true;
136 |
137 | m_Process.Exited += binary2textProcess_Exited;
138 |
139 | try
140 | {
141 | m_Process.Start();
142 | m_Process.BeginOutputReadLine();
143 | }
144 | catch( Exception e )
145 | {
146 | Debug.LogError( "Failed to start process : " + e.Message );
147 | }
148 | }
149 |
150 | private void binary2textProcess_Exited(object sender, EventArgs e)
151 | {
152 | m_Process.Close();
153 | m_Process.Dispose();
154 | m_Process = null;
155 |
156 | // TOOD wait while it exists
157 | if( File.Exists( m_BinaryFiles[m_FileIndex].FullName + ".txt" ) )
158 | m_TextFiles[m_FileIndex] = new FileInfo(m_BinaryFiles[m_FileIndex].FullName + ".txt");
159 | else
160 | Debug.LogError( "Could not find converted text file for " + m_BinaryFiles[m_FileIndex].Name + " : " + e.ToString() );
161 |
162 | m_FileIndex++;
163 | }
164 | }
165 | }
166 |
--------------------------------------------------------------------------------