Skip to content

Support 8.2 Root #30

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
YetAnotherBinarySpace wants to merge 4 commits into
base: master
Choose a base branch
from
Open

Support 8.2 Root #30

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
af4cd0a
Fixed #23
MaxtorCoder Apr 14, 2019
ee6bcad
Fixed footer reading.
MaxtorCoder Apr 15, 2019
506cd99
8.2 root change handling
YetAnotherBinarySpace Aug 26, 2019
0b4a5c3
Use listfile for files without hash names.
YetAnotherBinarySpace Oct 5, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions CASCEdit/CASContainer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,7 @@ public static void OpenEncoding()
}
}

public static void OpenRoot(LocaleFlags locale, uint minimumid = 0)
public static void OpenRoot(LocaleFlags locale, uint minimumid = 0, bool onlineListfile = false)
{
Logger.LogInformation("Loading Root...");

Expand All @@ -208,7 +208,7 @@ public static void OpenRoot(LocaleFlags locale, uint minimumid = 0)
if (idxInfo != null)
{
var path = Path.Combine(BasePath, "Data", "data", string.Format("data.{0:D3}", idxInfo.Archive));
RootHandler = new RootHandler(DataHandler.Read(path, idxInfo), locale, minimumid);
RootHandler = new RootHandler(DataHandler.Read(path, idxInfo), locale, minimumid, onlineListfile);
}
else
{
Expand All @@ -223,7 +223,7 @@ public static void OpenRoot(LocaleFlags locale, uint minimumid = 0)
Logger.LogCritical($"Unable to download Root {key}.");
}

RootHandler = new RootHandler(DataHandler.ReadDirect(path), locale, minimumid);
RootHandler = new RootHandler(DataHandler.ReadDirect(path), locale, minimumid, onlineListfile);
}
}

Expand Down
5 changes: 2 additions & 3 deletions CASCEdit/Handlers/ArchiveGroupIndex.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,12 +72,11 @@ public void Read()

Footer = new IndexFooter()
{
IndexBlockHash = br.ReadBytes(8),
TOCHash = br.ReadBytes(8),
Version = br.ReadByte(),
_11 = br.ReadByte(),
_12 = br.ReadByte(),
_13 = br.ReadByte(),
BlockSizeKb = br.ReadByte(),
Offset = br.ReadByte(),
Size = br.ReadByte(),
KeySize = br.ReadByte(),
Expand Down Expand Up @@ -155,7 +154,7 @@ public string Write()
bw.Write(Footer.Version);
bw.Write(Footer._11);
bw.Write(Footer._12);
bw.Write(Footer._13);
bw.Write(Footer.BlockSizeKb);
bw.Write(Footer.Offset);
bw.Write(Footer.Size);
bw.Write(Footer.KeySize);
Expand Down
46 changes: 23 additions & 23 deletions CASCEdit/Handlers/ArchiveIndex.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,23 +60,22 @@ private void Read()
br.BaseStream.Position += blockcount * 16;

//Block hashes - lower_md5 all blocks except last
br.BaseStream.Position += (blockcount - 1) * 8;
br.BaseStream.Position += blockcount * 8;

//Footer
Footer = new IndexFooter()
{
IndexBlockHash = br.ReadBytes(8),
TOCHash = br.ReadBytes(8),
Version = br.ReadByte(),
TOCHash = br.ReadBytes(8),
Version = br.ReadByte(),
_11 = br.ReadByte(),
_12 = br.ReadByte(),
_13 = br.ReadByte(),
BlockSizeKb = br.ReadByte(),
Offset = br.ReadByte(),
Size = br.ReadByte(),
KeySize = br.ReadByte(),
ChecksumSize = br.ReadByte(),
EntryCount = br.ReadUInt32(),
FooterMD5 = br.ReadBytes(8)
FooterMD5 = br.ReadBytes(Footer.ChecksumSize)
};
}
}
Expand All @@ -102,11 +101,11 @@ public string Write()

if (pos + entry.EntrySize > CHUNK_SIZE)
{
entryHashes.Add(Entries[i - 1].EKey.Value); //Last entry hash
entryHashes.Add(Entries[i - 1].EKey.Value); //Last entry hash

bw.Write(new byte[CHUNK_SIZE - pos]);
blockHashes.Add(GetBlockHash(bw, md5));
pos = 0;
bw.Write(new byte[CHUNK_SIZE - pos]);
blockHashes.Add(GetBlockHash(bw, md5));
pos = 0;
}

entry.Offset = offset;
Expand All @@ -130,23 +129,23 @@ public string Write()
//Block hashes
if (blockHashes.Count > 0)
{
bw.Write(blockHashes.SelectMany(x => x).ToArray());
blockHashes.Clear();
bw.Write(blockHashes.SelectMany(x => x).ToArray());
blockHashes.Clear();
}

//Footer Start
long posFooterStart = bw.BaseStream.Position;

//Calculate IndexBlockHash
bw.BaseStream.Position = CHUNK_SIZE * (blockCount - 1);
byte[] indexBlockHashBytes = new byte[CHUNK_SIZE];
bw.BaseStream.Read(indexBlockHashBytes, 0, indexBlockHashBytes.Length);
bw.BaseStream.Position = posFooterStart;
var lowerHash = md5.ComputeHash(indexBlockHashBytes).Take(8).ToArray();
bw.Write(lowerHash);
//Calculate IndexBlockHash
bw.BaseStream.Position = CHUNK_SIZE * (blockCount - 1);
byte[] indexBlockHashBytes = new byte[CHUNK_SIZE];
bw.BaseStream.Read(indexBlockHashBytes, 0, indexBlockHashBytes.Length);
bw.BaseStream.Position = posFooterStart;
var lowerHash = md5.ComputeHash(indexBlockHashBytes).Take(8).ToArray();
bw.Write(lowerHash);

//Calculate TOCHash
bw.BaseStream.Position = CHUNK_SIZE * blockCount;
//Calculate TOCHash
bw.BaseStream.Position = CHUNK_SIZE * blockCount;
byte[] tocHashBytes = new byte[8 + posFooterStart - bw.BaseStream.Position];
bw.BaseStream.Read(tocHashBytes, 0, tocHashBytes.Length);
var upperHash = md5.ComputeHash(tocHashBytes).Take(8).ToArray();
Expand All @@ -156,7 +155,7 @@ public string Write()
bw.Write(Footer.Version);
bw.Write(Footer._11);
bw.Write(Footer._12);
bw.Write(Footer._13);
bw.Write(Footer.BlockSizeKb);
bw.Write(Footer.Offset);
bw.Write(Footer.Size);
bw.Write(Footer.KeySize);
Expand Down Expand Up @@ -186,7 +185,8 @@ public string Write()
CASContainer.CDNConfig["archives"].RemoveAll(x => x == Path.GetFileNameWithoutExtension(BaseFile));
CASContainer.CDNConfig["archives"].Add(filename);
CASContainer.CDNConfig["archives"].Sort(new HashComparer());

CASContainer.Logger.LogInformation($"Index Hash Position: {CASContainer.CDNConfig["archives"].IndexOf(filename).ToString()} Index Size: {ms.Length}");
CASContainer.CDNConfig["archives-index-size"].Insert(CASContainer.CDNConfig["archives"].IndexOf(filename) - 1, ms.Length.ToString());
return path;
}
}
Expand Down
135 changes: 114 additions & 21 deletions CASCEdit/Handlers/RootHandler.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
using System.Threading.Tasks;
using System.Diagnostics;
using CASCEdit.IO;
using System.Net;

namespace CASCEdit.Handlers
{
Expand All @@ -24,20 +25,50 @@ public class RootHandler : IDisposable
private readonly uint minimumId;
private readonly EncodingMap encodingMap;

public RootHandler()
private int namedFiles = 0;
private int allFiles = 0;
private int parsedFiles = 0;
private const int headerMagic = 0x4D465354; // MFST


private Dictionary<uint, ulong> ListFile = new Dictionary<uint, ulong>();
private WebClient ListFileClient = new WebClient();


public RootHandler()
{
GlobalRoot = new RootChunk() { ContentFlags = ContentFlags.None, LocaleFlags = LocaleFlags.All_WoW };
encodingMap = new EncodingMap(EncodingType.ZLib, 9);
}

public RootHandler(Stream data, LocaleFlags locale, uint minimumid = 0)
public RootHandler(Stream data, LocaleFlags locale, uint minimumid = 0, bool onlineListfile = false)
{
this.minimumId = minimumid;
this.locale = locale;

BinaryReader stream = new BinaryReader(data);

long length = stream.BaseStream.Length;
string cdnPath = Helper.GetCDNPath("listfile.csv");

if (!(File.Exists(Path.Combine(CASContainer.Settings.OutputPath, cdnPath))) && onlineListfile)
{
CASContainer.Logger.LogInformation("Downloading listfile from WoW.Tools");
ListFileClient.DownloadFile("https://wow.tools/casc/listfile/download/csv/unverified", cdnPath);
}

BinaryReader stream = new BinaryReader(data);

// 8.2 root change
int magic = stream.ReadInt32();
bool newFormat = magic == headerMagic;
if (newFormat)
{
allFiles = stream.ReadInt32();
namedFiles = stream.ReadInt32();
}
else
{
stream.BaseStream.Position = 0;
}

long length = stream.BaseStream.Length;
while (stream.BaseStream.Position < length)
{
RootChunk chunk = new RootChunk()
Expand All @@ -47,8 +78,10 @@ public RootHandler(Stream data, LocaleFlags locale, uint minimumid = 0)
LocaleFlags = (LocaleFlags)stream.ReadUInt32(),
};

// set the global root
if (chunk.LocaleFlags == LocaleFlags.All_WoW && chunk.ContentFlags == ContentFlags.None)
parsedFiles += (int)chunk.Count;

// set the global root
if (chunk.LocaleFlags == LocaleFlags.All_WoW && chunk.ContentFlags == ContentFlags.None)
GlobalRoot = chunk;

uint fileDataIndex = 0;
Expand All @@ -66,14 +99,41 @@ public RootHandler(Stream data, LocaleFlags locale, uint minimumid = 0)
chunk.Entries.Add(entry);
}

foreach (var entry in chunk.Entries)
{
entry.CEKey = new MD5Hash(stream);
entry.NameHash = stream.ReadUInt64();
maxId = Math.Max(maxId, entry.FileDataId);
}

Chunks.Add(chunk);
if (newFormat)
{
foreach (var entry in chunk.Entries)
{
entry.CEKey = new MD5Hash(stream);
maxId = Math.Max(maxId, entry.FileDataId);
}

if (parsedFiles > allFiles - namedFiles)
{
foreach (var entry in chunk.Entries)
{
entry.NameHash = stream.ReadUInt64();
}
}
else // no namehash
{
foreach (var entry in chunk.Entries)
{
entry.NameHash = 0;
}
}

}
else
{
foreach (var entry in chunk.Entries)
{
entry.CEKey = new MD5Hash(stream);
entry.NameHash = stream.ReadUInt64();
maxId = Math.Max(maxId, entry.FileDataId);
}
}

Chunks.Add(chunk);
}

if (GlobalRoot == null)
Expand All @@ -82,8 +142,35 @@ public RootHandler(Stream data, LocaleFlags locale, uint minimumid = 0)
return;
}

// set maxid from cache
maxId = Math.Max(Math.Max(maxId, minimumid), CASContainer.Settings.Cache?.MaxId ?? 0);
// use listfile to assign names
var listFileLines = File.ReadAllLines(cdnPath);
foreach (var listFileData in listFileLines)
{
var splitData = listFileData.Split(';');

if (splitData.Length != 2)
continue;

if (!uint.TryParse(splitData[0], out uint listFileDataID))
continue;

ListFile[listFileDataID] = new Jenkins96().ComputeHash(splitData[1]);
}

foreach (var chunk in Chunks)
{
foreach (var entry in chunk.Entries)
{
if (entry.NameHash == 0)
{
if (ListFile.ContainsKey(entry.FileDataId))
entry.NameHash = ListFile[entry.FileDataId];
}
}
}

// set maxid from cache
maxId = Math.Max(Math.Max(maxId, minimumid), CASContainer.Settings.Cache?.MaxId ?? 0);

// store encoding map
encodingMap = (data as BLTEStream)?.EncodingMap.FirstOrDefault() ?? new EncodingMap(EncodingType.ZLib, 9);
Expand Down Expand Up @@ -120,7 +207,8 @@ public void AddEntry(string path, CASResult file)
.SelectMany(chunk => chunk.Entries) // Flatten the array to get all entries within all matching chunks
.Where(e => e.NameHash == namehash);

if (entries.Count() == 0) { // New file, we need to create an entry for it
if (entries.Count() == 0)
{ // New file, we need to create an entry for it
var cached = cache.Entries.FirstOrDefault(x => x.Path == path);
var fileDataId = Math.Max(maxId + 1, minimumId);

Expand All @@ -138,8 +226,13 @@ public void AddEntry(string path, CASResult file)

GlobalRoot.Entries.Add(entry); // Insert into the Global Root
maxId = Math.Max(entry.FileDataId, maxId); // Update the max id
} else { // Existing file, we just have to update the data hash
foreach (var entry in entries) {

cache?.AddOrUpdate(new CacheEntry(entry, file.EKey)); // If not done, sometimes files will not be added.
}
else
{ // Existing file, we just have to update the data hash
foreach (var entry in entries)
{
entry.CEKey = file.CEKey;
entry.Path = path;

Expand Down
3 changes: 1 addition & 2 deletions CASCEdit/Structs/IndexStructs.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,11 @@ public class IndexEntry

public class IndexFooter
{
public byte[] IndexBlockHash; // ChecksumSize
public byte[] TOCHash; // ChecksumSize
public byte Version = 1;
public byte _11 = 0;
public byte _12 = 0;
public byte _13 = 4;
public byte BlockSizeKb = 4;
public byte Offset = 4;
public byte Size = 4;
public byte KeySize = 16;
Expand Down
2 changes: 1 addition & 1 deletion CASCEdit/Structs/RootStructs.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ public enum ContentFlags : uint
F00000008 = 0x8, // added in 7.2.0.23436
F00000010 = 0x10, // added in 7.2.0.23436
LowViolence = 0x80, // many models have this flag
F10000000 = 0x10000000,
NoNames = 0x10000000,
F20000000 = 0x20000000, // added in 21737
Bundle = 0x40000000,
NoCompression = 0x80000000 // sounds have this flag
Expand Down
3 changes: 2 additions & 1 deletion CASCHost/AppSettings.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@ namespace CASCHost
public class AppSettings
{
public uint MinimumFileDataId { get; set; } // the minimum file id for new files
public bool BNetAppSupport { get; set; } = false; // create install and download files?
public bool OnlineListFile { get; set; } = false; // fetchs from WoW.Tools the lastest listfile
public bool BNetAppSupport { get; set; } = false; // create install and download files?
public bool StaticMode { get; set; } = false; // Build CDN file struct
public string RebuildPassword { get; set; } = "";

Expand Down
3 changes: 3 additions & 0 deletions CASCHost/CASCHost.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@
<PropertyGroup>
<TargetFramework>netcoreapp2.1</TargetFramework>
<Platforms>AnyCPU;x64</Platforms>
<ApplicationIcon />
<OutputType>Exe</OutputType>
<StartupObject />
</PropertyGroup>

<ItemGroup>
Expand Down
2 changes: 1 addition & 1 deletion CASCHost/DataWatcher.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ private void UpdateCASCDirectory(object obj)
CASContainer.Open(settings);
CASContainer.OpenCdnIndices(false);
CASContainer.OpenEncoding();
CASContainer.OpenRoot(settings.Locale, Startup.Settings.MinimumFileDataId);
CASContainer.OpenRoot(settings.Locale, Startup.Settings.MinimumFileDataId, Startup.Settings.OnlineListFile);

if(Startup.Settings.BNetAppSupport) // these are only needed by the bnet app launcher
{
Expand Down
Loading