├── .gitattributes
├── .gitignore
├── Utility
    ├── VirtualObject.cpp
    ├── Assert.inl
    ├── CMakeLists.txt
    ├── VirtualObject.hpp
    ├── MemoryMappedFile.cpp
    ├── RAIIWrapper.hpp
    ├── MemoryMappedFile_impl.hpp
    ├── MemoryMappedFile.hpp
    ├── Assert.hpp
    ├── DataBlock.hpp
    ├── MemoryMappedFile_impl.cpp
    └── Assert.cpp
├── LICENSE
├── FileFormats
    ├── CMakeLists.txt
    ├── Tlk.hpp
    ├── Erf.hpp
    ├── Key.hpp
    ├── 2da.hpp
    ├── Gff.hpp
    ├── Bif.hpp
    ├── Tlk
    │   ├── Tlk_Friendly.hpp
    │   ├── Tlk_Raw.cpp
    │   ├── Tlk_Raw.hpp
    │   └── Tlk_Friendly.cpp
    ├── Bif
    │   ├── Bif_Friendly.hpp
    │   ├── Bif_Friendly.cpp
    │   ├── Bif_Raw.cpp
    │   └── Bif_Raw.hpp
    ├── Erf
    │   ├── Erf_Friendly.hpp
    │   ├── Erf_Friendly.cpp
    │   ├── Erf_Raw.cpp
    │   └── Erf_Raw.hpp
    ├── Key
    │   ├── Key_Friendly.hpp
    │   ├── Key_Friendly.cpp
    │   ├── Key_Raw.cpp
    │   └── Key_Raw.hpp
    ├── 2da
    │   ├── 2da_Friendly.hpp
    │   ├── 2da_Raw.cpp
    │   ├── 2da_Raw.hpp
    │   └── 2da_Friendly.cpp
    ├── Resource.hpp
    ├── Gff
    │   ├── Gff_Friendly.hpp
    │   ├── Gff_Raw.cpp
    │   └── Gff_Raw.hpp
    └── Resource.cpp
├── Tools
    ├── CMakeLists.txt
    ├── Tool_ErfExtractor.cpp
    ├── Tool_2daMerge.cpp
    ├── Tool_KeyBifExtractor.cpp
    ├── Tool_GeneratePlaceableBlueprints.cpp
    └── Tool_DiffCreature.cpp
├── Examples
    ├── CMakeLists.txt
    ├── Example_Tlk.cpp
    ├── Example_Bif.cpp
    ├── Example_Erf.cpp
    ├── Example_Key.cpp
    ├── Example_2da.cpp
    └── Example_Gff.cpp
├── README
└── CMakeLists.txt


/.gitattributes:
--------------------------------------------------------------------------------
1 | * text eol=lf
2 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | Build-*/*
2 | x64/*


--------------------------------------------------------------------------------
/Utility/VirtualObject.cpp:
--------------------------------------------------------------------------------
1 | #include "Utility/VirtualObject.hpp"
2 | 
3 | VirtualObject::~VirtualObject() {}
4 | 


--------------------------------------------------------------------------------
/Utility/Assert.inl:
--------------------------------------------------------------------------------
1 | template <typename ... Args>
2 | void Fail(const char* condition, const char* file, int line, const char* format, Args ... args)
3 | {
4 |     char buffer[1536];
5 |     std::sprintf(buffer, format, args ...);
6 |     Fail(condition, file, line, buffer);
7 | }


--------------------------------------------------------------------------------
/Utility/CMakeLists.txt:
--------------------------------------------------------------------------------
1 | add_library(Utility STATIC
2 |     Assert.cpp Assert.hpp Assert.inl
3 |     DataBlock.hpp
4 |     MemoryMappedFile.cpp MemoryMappedFile.hpp
5 |     MemoryMappedFile_impl.cpp MemoryMappedFile_impl.hpp
6 |     RAIIWrapper.hpp
7 |     VirtualObject.cpp VirtualObject.hpp)
8 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 |         DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
 2 |                     Version 2, December 2004
 3 | 
 4 |  Copyright (C) 2004 Sam Hocevar <sam@hocevar.net>
 5 | 
 6 |  Everyone is permitted to copy and distribute verbatim or modified
 7 |  copies of this license document, and changing it is allowed as long
 8 |  as the name is changed.
 9 | 
10 |             DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
11 |    TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
12 | 
13 |   0. You just DO WHAT THE FUCK YOU WANT TO.
14 | 


--------------------------------------------------------------------------------
/Utility/VirtualObject.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | // The purpose of this structure is simple. Sometimes, you just want to store a unique pointer to -something-
 4 | // to wrap the lifetime of resources.
 5 | //
 6 | // You don't really care what - you just need a base class for it.
 7 | //
 8 | // The only requirement is for this base class to have a virtual destructor so when it is freed, the
 9 | // resources of whatever it owns are also freed.
10 | //
11 | // This is that structure.
12 | 
13 | struct VirtualObject
14 | {
15 |     virtual ~VirtualObject() = 0;
16 | };
17 | 


--------------------------------------------------------------------------------
/Utility/MemoryMappedFile.cpp:
--------------------------------------------------------------------------------
 1 | #include "Utility/MemoryMappedFile.hpp"
 2 | #include "Utility/Assert.hpp"
 3 | #include "Utility/MemoryMappedFile_impl.hpp"
 4 | 
 5 | MemoryMappedFile::MemoryMappedFile() { }
 6 | MemoryMappedFile::MemoryMappedFile(MemoryMappedFile&& rhs) : m_DataBlock(std::move(rhs.m_DataBlock)), m_PlatformImpl(std::move(rhs.m_PlatformImpl)) { }
 7 | MemoryMappedFile::~MemoryMappedFile() { }
 8 | 
 9 | bool MemoryMappedFile::MemoryMap(char const* path, MemoryMappedFile* out)
10 | {
11 |     out->m_PlatformImpl = std::make_unique<MemoryMappedFile_impl>();
12 |     return out->m_PlatformImpl->Map(path, &out->m_DataBlock);
13 | }
14 | 
15 | NonOwningDataBlock const& MemoryMappedFile::GetDataBlock()
16 | {
17 |     return m_DataBlock;
18 | }
19 | 


--------------------------------------------------------------------------------
/Utility/RAIIWrapper.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include "Utility/VirtualObject.hpp"
 4 | 
 5 | // An RAII wrapper is a simple wrapper that takes moved-in object and stores it.
 6 | // That's all it does.
 7 | // The hook is that it inherits from a VirtualObject - so you can store anything in this
 8 | // thing, and keep a unique_ptr to the VirtualObject. This is powerful because you can
 9 | // have a class which can store different types of objects based on internal implementation
10 | // details and then use RAII to free the one that is used automatically.
11 | 
12 | template <typename T>
13 | struct RAIIWrapper : public VirtualObject
14 | {
15 |     T m_Data;
16 |     RAIIWrapper(T&& data) : m_Data(std::forward<T>(data)) { }
17 |     virtual ~RAIIWrapper() { }
18 | };
19 | 


--------------------------------------------------------------------------------
/Utility/MemoryMappedFile_impl.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include "Utility/DataBlock.hpp"
 4 | 
 5 | #if OS_WINDOWS
 6 |     #include "Windows.h"
 7 | #else
 8 |     #include <fcntl.h>
 9 |     #include <sys/mman.h>
10 |     #include <sys/stat.h>
11 |     #include <unistd.h>
12 | #endif
13 | 
14 | // This is separated out into an _impl to avoid drawing in system headers for consumers of the API.
15 | class MemoryMappedFile_impl
16 | {
17 | public:
18 |     MemoryMappedFile_impl();
19 |     ~MemoryMappedFile_impl();
20 | 
21 |     bool Map(char const* path, NonOwningDataBlock* out);
22 | 
23 | private:
24 | 
25 | #if OS_WINDOWS
26 |     HANDLE m_File;
27 |     HANDLE m_MemoryMap;
28 |     LPVOID m_Ptr;
29 | #else
30 |     int m_FileDescriptor;
31 |     void* m_Ptr;
32 |     std::size_t m_PtrLength;
33 | #endif
34 | };
35 | 


--------------------------------------------------------------------------------
/FileFormats/CMakeLists.txt:
--------------------------------------------------------------------------------
 1 | add_library(FileFormats STATIC
 2 | 
 3 |     Bif.hpp
 4 |     Bif/Bif_Raw.cpp Bif/Bif_Raw.hpp
 5 |     Bif/Bif_Friendly.cpp Bif/Bif_Friendly.hpp
 6 | 
 7 |     Erf.hpp
 8 |     Erf/Erf_Raw.cpp Erf/Erf_Raw.hpp
 9 |     Erf/Erf_Friendly.cpp Erf/Erf_Friendly.hpp
10 | 
11 |     Gff.hpp
12 |     Gff/Gff_Raw.cpp Gff/Gff_Raw.hpp
13 |     Gff/Gff_Friendly.cpp Gff/Gff_Friendly.hpp
14 | 
15 |     Key.hpp
16 |     Key/Key_Raw.cpp Key/Key_Raw.hpp
17 |     Key/Key_Friendly.cpp Key/Key_Friendly.hpp
18 | 
19 |     Tlk.hpp
20 |     Tlk/Tlk_Raw.cpp Tlk/Tlk_Raw.hpp
21 |     Tlk/Tlk_Friendly.cpp Tlk/Tlk_Friendly.hpp
22 | 
23 |     2da.hpp
24 |     2da/2da_Raw.cpp 2da/2da_Raw.hpp
25 |     2da/2da_Friendly.cpp 2da/2da_Friendly.hpp
26 | 
27 |     Resource.cpp Resource.hpp
28 | )
29 | 
30 | target_link_libraries(FileFormats Utility)
31 | 


--------------------------------------------------------------------------------
/Utility/MemoryMappedFile.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include "Utility/DataBlock.hpp"
 4 | 
 5 | #include <cstddef>
 6 | #include <memory>
 7 | 
 8 | class MemoryMappedFile_impl;
 9 | 
10 | // This class wraps memory-mapped read only access to a file.
11 | // This will map the entire range of the file (0 -> fileSize) into a memory rage represented by GetDataBlock().
12 | // This can later be expanded to support spans within the file and writing too.
13 | class MemoryMappedFile
14 | {
15 | public:
16 |     MemoryMappedFile();
17 |     MemoryMappedFile(MemoryMappedFile&& rhs);
18 |     ~MemoryMappedFile();
19 | 
20 |     static bool MemoryMap(char const* path, MemoryMappedFile* out);
21 |     NonOwningDataBlock const& GetDataBlock();
22 | 
23 | private:
24 |     NonOwningDataBlock m_DataBlock;
25 |     std::unique_ptr<MemoryMappedFile_impl> m_PlatformImpl;
26 | };
27 | 


--------------------------------------------------------------------------------
/Tools/CMakeLists.txt:
--------------------------------------------------------------------------------
 1 | add_executable(2da_merge Tool_2daMerge.cpp)
 2 | target_link_libraries(2da_merge FileFormats)
 3 | set_target_properties(2da_merge PROPERTIES FOLDER "Tools")
 4 | 
 5 | add_executable(diff_creature "Tool_DiffCreature.cpp")
 6 | target_link_libraries(diff_creature FileFormats)
 7 | set_target_properties(diff_creature PROPERTIES FOLDER "Tools")
 8 | 
 9 | add_executable(erf_extractor Tool_ErfExtractor.cpp)
10 | target_link_libraries(erf_extractor FileFormats)
11 | set_target_properties(erf_extractor PROPERTIES FOLDER "Tools")
12 | 
13 | add_executable(key_bif_extractor Tool_KeyBifExtractor.cpp)
14 | target_link_libraries(key_bif_extractor FileFormats)
15 | set_target_properties(key_bif_extractor PROPERTIES FOLDER "Tools")
16 | 
17 | add_executable(generate_placeable_blueprints Tool_GeneratePlaceableBlueprints.cpp)
18 | target_link_libraries(generate_placeable_blueprints FileFormats)
19 | set_target_properties(generate_placeable_blueprints PROPERTIES FOLDER "Tools")
20 | 


--------------------------------------------------------------------------------
/Examples/CMakeLists.txt:
--------------------------------------------------------------------------------
 1 | add_executable(Example_2da Example_2da.cpp)
 2 | target_link_libraries(Example_2da FileFormats)
 3 | set_target_properties(Example_2da PROPERTIES FOLDER "Examples")
 4 | 
 5 | add_executable(Example_Bif Example_Bif.cpp)
 6 | target_link_libraries(Example_Bif FileFormats)
 7 | set_target_properties(Example_Bif PROPERTIES FOLDER "Examples")
 8 | 
 9 | add_executable(Example_Erf Example_Erf.cpp)
10 | target_link_libraries(Example_Erf FileFormats)
11 | set_target_properties(Example_Erf PROPERTIES FOLDER "Examples")
12 | 
13 | add_executable(Example_Gff Example_Gff.cpp)
14 | target_link_libraries(Example_Gff FileFormats)
15 | set_target_properties(Example_Gff PROPERTIES FOLDER "Examples")
16 | 
17 | add_executable(Example_Key Example_Key.cpp)
18 | target_link_libraries(Example_Key FileFormats)
19 | set_target_properties(Example_Key PROPERTIES FOLDER "Examples")
20 | 
21 | add_executable(Example_Tlk Example_Tlk.cpp)
22 | target_link_libraries(Example_Tlk FileFormats)
23 | set_target_properties(Example_Tlk PROPERTIES FOLDER "Examples")
24 | 


--------------------------------------------------------------------------------
/FileFormats/Tlk.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | // This file provides read-only access to TLK data.
 4 | // In the FileFormats::Tlk::Raw namespace is located Tlk, which wraps the raw data structure.
 5 | // In the FileFormats::Tlk::Friendly namespace is located Tlk, which exposes a much more user friendly structure.
 6 | //
 7 | // How to use:
 8 | //
 9 | // Step 1: Load your TFF file into memory.
10 | // Step 2: Construct a Tlk as such: FileFormats::Tlk::Raw::Tlk::ReadFromBytes(bytes);
11 | // Step 3: If user friendly access to fields is desired, construct a Tlk from FileFormats::Tlk::Friendly::Tlk(rawTlk).
12 | // - You can get entries via operator[].
13 | // - Use begin/end() (or ranged-based loop) to iterate all entries.
14 | //
15 | // For further information refer to https://wiki.neverwintervault.org/pages/viewpage.action?pageId=327727
16 | // Specifically, https://wiki.neverwintervault.org/download/attachments/327727/Bioware_Aurora_TalkTable_Format.pdf?api=v2
17 | 
18 | #include "FileFormats/Tlk/Tlk_Raw.hpp"
19 | #include "FileFormats/Tlk/Tlk_Friendly.hpp"
20 | 


--------------------------------------------------------------------------------
/FileFormats/Erf.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | // This file provides read-only access to ERF data.
 4 | // In the FileFormats::Erf::Raw namespace is located Erf, which wraps the raw data structure.
 5 | // In the FileFormats::Erf::Friendly namespace is located Erf, which exposes a much more user friendly structure.
 6 | //
 7 | // How to use:
 8 | //
 9 | // Step 1: Load your ERF file into memory.
10 | // Step 2: Construct a Erf as such: FileFormats::Erf::Raw::Erf::ReadFromBytes(bytes, totalBytes);
11 | // Step 3: If user friendly access is desired, construct a Erf from FileFormats::Erf::Friendly::Erf(rawErf).
12 | // - Localised descriptions can be accessed by .GetDescriptions().
13 | // - Resources can be accessed by .GetResources().
14 | // - Refer to Example_Erf.cpp if the usage is unclear.
15 | //
16 | // For further information refer to https://wiki.neverwintervault.org/pages/viewpage.action?pageId=327727
17 | // Specifically, https://wiki.neverwintervault.org/download/attachments/327727/Bioware_Aurora_ERF_Format.pdf?api=v2
18 | 
19 | #include "FileFormats/Erf/Erf_Raw.hpp"
20 | #include "FileFormats/Erf/Erf_Friendly.hpp"
21 | 


--------------------------------------------------------------------------------
/Utility/Assert.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <cstdio>
 4 | 
 5 | namespace Assert {
 6 | 
 7 | #if TAR_RELEASE
 8 |     #define ASSERT(condition) (void)0
 9 |     #define ASSERT_MSG(condition, format, ...) (void)0
10 |     #define ASSERT_FAIL() (void)0
11 |     #define ASSERT_FAIL_MSG(format, ...) (void)0
12 | #else
13 |     #define ASSERT(condition) \
14 |         if (!(condition)) ::Assert::Fail((#condition), __FILE__, __LINE__, nullptr)
15 |     #define ASSERT_MSG(condition, format, ...) \
16 |         if (!(condition)) ::Assert::Fail((#condition), __FILE__, __LINE__, (format), ##__VA_ARGS__)
17 |     #define ASSERT_FAIL() \
18 |         ::Assert::Fail(nullptr, __FILE__, __LINE__, nullptr)
19 |     #define ASSERT_FAIL_MSG(format, ...) \
20 |         ::Assert::Fail(nullptr, __FILE__, __LINE__, (format), ##__VA_ARGS__)
21 | #endif
22 | 
23 | void Fail(const char* condition, const char* file, int line, const char* message);
24 | 
25 | template <typename ... Args>
26 | void Fail(const char* condition, const char* file, int line, const char* format, Args ... args);
27 | 
28 | #include "Utility/Assert.inl"
29 | 
30 | }
31 | 


--------------------------------------------------------------------------------
/FileFormats/Key.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | // This file provides read-only access to KEY data.
 4 | // In the FileFormats::Key::Raw namespace is located Key, which wraps the raw data structure.
 5 | // In the FileFormats::Key::Friendly namespace is located Key, which exposes a much more user friendly structure.
 6 | //
 7 | // How to use:
 8 | //
 9 | // Step 1: Load your KEY file into memory.
10 | // Step 2: Construct a Key as such: FileFormats::Key::Raw::Key::ReadFromBytes(bytes);
11 | // Step 3: If user friendly access is desired, construct a Key from FileFormats::Key::Friendly::Key(rawKey).
12 | // - Referenced BIFs can be accessed by .GetReferencedBifs().
13 | // - Referenced resources can be accessed by .GetReferencedResources().
14 | // - Refer to Example_Key.cpp if the usage is unclear.
15 | //
16 | // For further information refer to https://wiki.neverwintervault.org/pages/viewpage.action?pageId=327727
17 | // Specifically, https://wiki.neverwintervault.org/download/attachments/327727/Bioware_Aurora_KeyBIF_Format.pdf?api=v2
18 | 
19 | #include "FileFormats/Key/Key_Raw.hpp"
20 | #include "FileFormats/Key/Key_Friendly.hpp"
21 | 


--------------------------------------------------------------------------------
/FileFormats/2da.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | // This file provides read-only access to 2DA data.
 4 | // In the FileFormats::TwoDA::Raw namespace is located TwoDA, which wraps the raw data structure.
 5 | // In the FileFormats::TwoDA::Friendly namespace is located TwoDA, which exposes a much more user friendly structure.
 6 | //
 7 | // How to use:
 8 | //
 9 | // Step 1: Load your 2DA file into memory.
10 | // Step 2: Construct a TwoDa as such: FileFormats::TwoDa::Raw::TwoDa::ReadFromBytes(bytes, totalBytes);
11 | // Step 3: If user friendly access is desired, construct a TwoDa from FileFormats::TwoDa::Friendly::TwoDa(raw2Da).
12 | // - You can access rows and columns directly: twoda[0]["LABEL"];
13 | // - You can iterate over the collection: refer to Example_2da.cpp.
14 | // - You can extract the string, int, or float representation with the appropriate functions.
15 | //
16 | // For further information refer to https://wiki.neverwintervault.org/pages/viewpage.action?pageId=327727
17 | // Specifically, https://wiki.neverwintervault.org/download/attachments/327727/Bioware_Aurora_2DA_Format.pdf?api=v2
18 | 
19 | #include "FileFormats/2da/2da_Raw.hpp"
20 | #include "FileFormats/2da/2da_Friendly.hpp"
21 | 


--------------------------------------------------------------------------------
/FileFormats/Gff.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | // This file provides read-only access to GFF data.
 4 | // In the FileFormats::Gff::Raw namespace is located Gff, which wraps the raw data structure.
 5 | // In the FileFormats::Gff::Friendly namespace is located Gff, which exposes a much more user friendly structure.
 6 | //
 7 | // How to use:
 8 | //
 9 | // Step 1: Load your GFF file into memory.
10 | // Step 2: Construct a Gff as such: FileFormats::Gff::Raw::Gff::ReadFromBytes(bytes);
11 | // - You can browse the loaded field format and extract fields using the ConstructX functions.
12 | // Step 3: If user friendly access to fields is desired, construct a Gff from FileFormats::Gff::Friendly::Gff(rawGff).
13 | // - You can access the top level struct with GetTopLevelStruct().
14 | // - You can access fields with GetTopLevelStruct().ReadField<Type_CExoString>("FIELD_NAME").
15 | //
16 | // For further information refer to https://wiki.neverwintervault.org/pages/viewpage.action?pageId=327727
17 | // Specifically, https://wiki.neverwintervault.org/download/attachments/327727/Bioware_Aurora_GFF_Format.pdf?api=v2
18 | 
19 | #include "FileFormats/Gff/Gff_Raw.hpp"
20 | #include "FileFormats/Gff/Gff_Friendly.hpp"
21 | 


--------------------------------------------------------------------------------
/FileFormats/Bif.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | // This file provides read-only access to BIF data.
 4 | // In the FileFormats::Bif::Raw namespace is located Bif, which wraps the raw data structure.
 5 | // In the FileFormats::Bif::Friendly namespace is located Bif, which exposes a much more user friendly structure.
 6 | //
 7 | // How to use:
 8 | //
 9 | // Step 1: Load your BIF file into memory.
10 | // Step 2: Construct a Bif as such: FileFormats::Bif::Raw::Bif::ReadFromBytes(bytes, totalBytes);
11 | // Step 3: If user friendly access is desired, construct a Bif from FileFormats::Bif::Friendly::Bif(rawBif).
12 | // - The resources can be accessed with .GetResources(). They are bucketed as such - resources[id] -> type / data.
13 | // - Note that we ignore the fixed resource table in the friendly implementation.
14 | // - Refer to Example_Bif.cpp if the usage is unclear.
15 | //
16 | // For further information refer to https://wiki.neverwintervault.org/pages/viewpage.action?pageId=327727
17 | // Specifically, https://wiki.neverwintervault.org/download/attachments/327727/Bioware_Aurora_KeyBIF_Format.pdf?api=v2
18 | 
19 | #include "FileFormats/Bif/Bif_Raw.hpp"
20 | #include "FileFormats/Bif/Bif_Friendly.hpp"
21 | 


--------------------------------------------------------------------------------
/FileFormats/Tlk/Tlk_Friendly.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include "FileFormats/Tlk/Tlk_Raw.hpp"
 4 | 
 5 | #include <cstddef>
 6 | #include <map>
 7 | #include <optional>
 8 | #include <string>
 9 | 
10 | namespace FileFormats::Tlk::Friendly {
11 | 
12 | using StrRef = std::uint32_t;
13 | 
14 | struct TlkEntry
15 | {
16 |     std::optional<std::string> m_String;
17 |     std::optional<std::string> m_SoundResRef;
18 |     std::optional<float> m_SoundLength;
19 | };
20 | 
21 | class Tlk
22 | {
23 | public:
24 |     Tlk(Raw::Tlk const& rawKey);
25 | 
26 |     // We use a map rather than unordered_map here because it's more user friendly to iterate from 0 -> max.
27 |     using TlkMapType = std::map<StrRef, TlkEntry>;
28 | 
29 |     // Returns the string associated with the strref, or empty string ("").
30 |     std::string const& operator[](StrRef strref) const;
31 | 
32 |     TlkEntry* Get(StrRef strref) const;
33 |     void Set(StrRef strref, TlkEntry value);
34 | 
35 |     std::uint32_t GetLanguageId() const;
36 |     void SetLanguageId(std::uint32_t id);
37 | 
38 |     TlkMapType::const_iterator begin() const;
39 |     TlkMapType::const_iterator end() const;
40 | 
41 |     bool WriteToFile(const char* path) const;
42 | 
43 | private:
44 |     std::uint32_t m_LanguageId;
45 |     TlkMapType m_TlkMap;
46 | };
47 | 
48 | }
49 | 


--------------------------------------------------------------------------------
/Utility/DataBlock.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <cstddef>
 4 | #include <vector>
 5 | 
 6 | // This describes a block of data - like an array_view or a span<t> or something like that.
 7 | // It can be an owning or non-owning.
 8 | struct DataBlock
 9 | {
10 |     // Returns a non-owning pointer to the data.
11 |     virtual std::byte const* GetData() const = 0;
12 | 
13 |     // Returns the length of the data.
14 |     virtual std::size_t GetDataLength() const  = 0;
15 | 
16 |     virtual ~DataBlock() {};
17 | };
18 | 
19 | struct OwningDataBlock : public DataBlock
20 | {
21 |     // The raw data associated with this.
22 |     std::vector<std::byte> m_Data;
23 | 
24 |     virtual std::byte const* GetData() const override { return m_Data.data(); }
25 |     virtual std::size_t GetDataLength() const override { return m_Data.size(); }
26 |     virtual ~OwningDataBlock() {};
27 | };
28 | 
29 | struct NonOwningDataBlock : public DataBlock
30 | {
31 |     // The raw data associated with this. This is a NON-OWNING pointer.
32 |     std::byte const* m_Data;
33 | 
34 |     // And the length.
35 |     std::size_t m_DataLength;
36 | 
37 |     virtual std::byte const* GetData() const override { return m_Data; }
38 |     virtual std::size_t GetDataLength() const override { return m_DataLength; }
39 |     virtual ~NonOwningDataBlock() {};
40 | };
41 | 


--------------------------------------------------------------------------------
/README:
--------------------------------------------------------------------------------
 1 | This is a cross-platform library written in C++ which allows developers to read many of the critical file types used by Neverwinter Nights 1.69 / 1.74. It compiles and runs on Windows and Linux on x86 processors. There will probably be issues on ARM processors.
 2 | 
 3 | Requirements: A compliant C++17 compiler is required. For Windows, this is Visual Studio Preview 1 or newer. I use GCC 7.2 on Linux but an older version may or may not work.
 4 | 
 5 | Check the examples folder to see how the library is used.
 6 | 
 7 | The supported file types are:
 8 | 
 9 | - GFF (bic, itp, etc)
10 | - ERF (hak, mod, sav, etc)
11 | - KEY
12 | - BIF
13 | - TLK
14 | 
15 | The library is completely free for anyone to copy, use, and change with or without attribution.
16 | 
17 | There are some tools in the Tools subdirectory:
18 | 
19 | - 2da_merge allows merging a 2da into a base 2da, overwriting rows that are already present or inserting ones that are not.
20 | - diff_creature diffs two creature GFF files and produces a report showing any changes to key fields (like attributes, HP, AC, or local variables).
21 | - generate_placeable_blueprints allows the user to generate a series of blueprints from placeables defined in 2da using a base blueprint
22 | - key_bif_extractor allows extracting all resources in a KEY from their BIFs.
23 | - erf_extractor allows extracting all resources from an ERF.
24 | 


--------------------------------------------------------------------------------
/FileFormats/Bif/Bif_Friendly.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include "FileFormats/Bif/Bif_Raw.hpp"
 4 | 
 5 | #include <memory>
 6 | #include <optional>
 7 | #include <unordered_map>
 8 | 
 9 | namespace FileFormats::Bif::Friendly {
10 | 
11 | struct BifResource
12 | {
13 |     // The actual ID listed in the BIF.
14 |     std::uint32_t m_ResId;
15 | 
16 |     // The resource type.
17 |     Resource::ResourceType m_ResType;
18 | 
19 |     // The underlying data for this resource.
20 |     std::unique_ptr<DataBlock> m_DataBlock;
21 | };
22 | 
23 | // This is a user friendly wrapper around the Bif data.
24 | // NOTE: We ignore the fixed resource table as it is not implemented per the spec.
25 | class Bif
26 | {
27 | public:
28 |     // This constructs a friendly BIF from a raw BIF.
29 |     Bif(Raw::Bif const& rawBif);
30 | 
31 |     // This constructs a friendly BIF from a raw BIF whose ownership has been passed to us.
32 |     // If ownership of the Bif is passed to us, we construct streamed resources, thus lowering
33 |     // the memory usage significantly.
34 |     Bif(Raw::Bif&& rawBif);
35 | 
36 |     using BifResourceMap = std::unordered_map<std::uint32_t, BifResource>;
37 |     BifResourceMap const& GetResources() const;
38 | 
39 | private:
40 |     std::optional<Raw::Bif> m_RawBif;
41 | 
42 |     void ConstructInternal(Raw::Bif const& rawBif);
43 | 
44 |     // This maps between ID -> { BifResource }
45 |     BifResourceMap m_Resources;
46 | };
47 | 
48 | }
49 | 


--------------------------------------------------------------------------------
/Tools/Tool_ErfExtractor.cpp:
--------------------------------------------------------------------------------
 1 | #include "FileFormats/Erf.hpp"
 2 | #include "Utility/Assert.hpp"
 3 | 
 4 | namespace
 5 | {
 6 | 
 7 | int extract_erf(const char* out_path, const char* in_path)
 8 | {
 9 |     using namespace FileFormats::Erf;
10 | 
11 |     Raw::Erf erf_raw;
12 |     bool loaded = Raw::Erf::ReadFromFile(in_path, &erf_raw);
13 | 
14 |     if (!loaded)
15 |     {
16 |         std::printf("Failed to open %s.", in_path);
17 |         return 1;
18 |     }
19 | 
20 |     Friendly::Erf erf(std::move(erf_raw));
21 | 
22 |     for (const Friendly::ErfResource& resource : erf.GetResources())
23 |     {
24 |         char path[512];
25 |         sprintf(path, "%s/%s.%s", out_path, resource.m_ResRef.c_str(), FileFormats::Resource::StringFromResourceType(resource.m_ResType));
26 | 
27 |         FILE* file = std::fopen(path, "wb");
28 |         if (file)
29 |         {
30 |             std::printf("Writing %s.\n", path);
31 |             std::fwrite(resource.m_DataBlock->GetData(), resource.m_DataBlock->GetDataLength(), 1, file);
32 |             std::fclose(file);
33 |         }
34 |         else
35 |         {
36 |             std::printf("Failed to open %s\n", path);
37 |         }
38 |     }
39 | 
40 |     return 0;
41 | }
42 | 
43 | }
44 | 
45 | int main(int argc, char** argv)
46 | {
47 |     if (argc != 3)
48 |     {
49 |         std::printf("erf_extractor [out_dir] [erf_path]\n");
50 |         return 1;
51 |     }
52 | 
53 |     return extract_erf(argv[1], argv[2]);
54 | }
55 | 


--------------------------------------------------------------------------------
/FileFormats/Erf/Erf_Friendly.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include "FileFormats/Erf/Erf_Raw.hpp"
 4 | 
 5 | #include <optional>
 6 | 
 7 | namespace FileFormats::Erf::Friendly {
 8 | 
 9 | struct ErfResource
10 | {
11 |     // The file name of the resource.
12 |     std::string m_ResRef;
13 | 
14 |     // The type of the resource.
15 |     Resource::ResourceType m_ResType;
16 | 
17 |     // This is mostly redundant - but could be useful somewhere.
18 |     std::uint32_t m_ResourceId;
19 | 
20 |     // The underlying data for this resource.
21 |     std::unique_ptr<DataBlock> m_DataBlock;
22 | };
23 | 
24 | // This is a user friendly wrapper around the Erf data.
25 | class Erf
26 | {
27 | public:
28 |     // This constructs a friendly Erf from a raw Erf.
29 |     Erf(Raw::Erf const& rawBif);
30 | 
31 |     // This constructs a friendly Erf from a raw Erf whose ownership has been passed to us.
32 |     // If ownership of the Erf is passed to us, we construct streamed resources, thus lowering
33 |     // the memory usage significantly.
34 |     Erf(Raw::Erf&& rawBif);
35 | 
36 |     std::vector<Raw::ErfLocalisedString> const& GetDescriptions() const;
37 |     std::vector<ErfResource> const& GetResources() const;
38 | 
39 | private:
40 |     std::optional<Raw::Erf> m_RawErf;
41 | 
42 |     void ConstructInternal(Raw::Erf const& rawErf);
43 | 
44 |     // This is a vector of localised descriptions for the ERF resource.
45 |     std::vector<Raw::ErfLocalisedString> m_Descriptions;
46 | 
47 |     // A vector of resources contained within this ERF.
48 |     std::vector<ErfResource> m_Resources;
49 | };
50 | 
51 | }
52 | 


--------------------------------------------------------------------------------
/Examples/Example_Tlk.cpp:
--------------------------------------------------------------------------------
 1 | #include "FileFormats/Tlk.hpp"
 2 | #include "Utility/Assert.hpp"
 3 | 
 4 | namespace {
 5 | 
 6 | int TlkExample(char* path);
 7 | 
 8 | int TlkExample(char* path)
 9 | {
10 |     using namespace FileFormats::Tlk;
11 | 
12 |     Raw::Tlk rawTlk;
13 |     bool loaded = Raw::Tlk::ReadFromFile(path, &rawTlk);
14 | 
15 |     std::printf("Tlk FileType: %.4s\n", rawTlk.m_Header.m_FileType);
16 |     std::printf("Tlk FileVersion: %.4s\n", rawTlk.m_Header.m_FileVersion);
17 | 
18 |     if (!loaded)
19 |     {
20 |         std::printf("Failed to load the Tlk file. Check the FileType and FileVersion and ensure the file is well formed.\n");
21 |         return 1;
22 |     }
23 | 
24 |     Friendly::Tlk tlk(std::move(rawTlk));
25 | 
26 |     // Grab some strings via direct strref. If it doesn't exist, it returns emptry string.
27 |     std::printf("\n0x00000000: '%s'", tlk[0x00000000].c_str());
28 |     std::printf("\n0x00000010: '%s'", tlk[0x00000010].c_str());
29 |     std::printf("\n0xFFFFFFFF: '%s'", tlk[0xFFFFFFFF].c_str());
30 | 
31 |     // Print the entire tlk table.
32 |     for (auto const& entry : tlk)
33 |     {
34 |         std::printf("\n%u -> '%s'", entry.first, entry.second.m_String.value_or("****").c_str());
35 |     }
36 | 
37 |     // Then save the tlk back out next to the original.
38 |     char pathBuffer[1024];
39 |     std::sprintf(pathBuffer, "%s.1", path);
40 |     bool written = tlk.WriteToFile(pathBuffer);
41 |     ASSERT(written);
42 | 
43 |     return 0;
44 | }
45 | 
46 | }
47 | 
48 | int main(int argc, char** argv)
49 | {
50 |     ASSERT(argc == 2);
51 |     return TlkExample(argv[1]);
52 | }
53 | 


--------------------------------------------------------------------------------
/FileFormats/Key/Key_Friendly.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include "FileFormats/Key/Key_Raw.hpp"
 4 | #include <string>
 5 | 
 6 | namespace FileFormats::Key::Friendly {
 7 | 
 8 | struct KeyBifReference
 9 | {
10 |     // When m_Drives == 0, this refers to the installation directory of the app.
11 |     // When m_Drives != 0, this is implementation defined.
12 |     std::uint16_t m_Drives;
13 | 
14 |     // The path (relative to the root of the drive above) of the BIF.
15 |     std::string m_Path;
16 | 
17 |     // Total byte size of the BIF.
18 |     std::uint32_t m_FileSize;
19 | };
20 | 
21 | struct KeyBifReferencedResource
22 | {
23 |     // The file name of the resource.
24 |     std::string m_ResRef;
25 | 
26 |     // The type of the resource.
27 |     Resource::ResourceType m_ResType;
28 | 
29 |     // The unmodified ResID of the resource.
30 |     std::uint32_t m_ResId;
31 | 
32 |     // The referenced res ID of the resource.
33 |     // This ID should match up with ID indexed into the BIF map.
34 |     std::uint32_t m_ReferencedBifResId;
35 | 
36 |     // The index into m_ReferencedBifs that this resource is inside.
37 |     std::size_t m_ReferencedBifIndex;
38 | };
39 | 
40 | // This is a user friendly wrapper around the Key data.
41 | class Key
42 | {
43 | public:
44 |     Key(Raw::Key const& rawKey);
45 | 
46 |     std::vector<KeyBifReference> const& GetReferencedBifs() const;
47 |     std::vector<KeyBifReferencedResource> const& GetReferencedResources() const;
48 | 
49 | private:
50 |     std::vector<KeyBifReference> m_ReferencedBifs;
51 |     std::vector<KeyBifReferencedResource> m_ReferencedResources;
52 | };
53 | 
54 | }
55 | 


--------------------------------------------------------------------------------
/Examples/Example_Bif.cpp:
--------------------------------------------------------------------------------
 1 | #include "FileFormats/Bif.hpp"
 2 | #include "Utility/Assert.hpp"
 3 | 
 4 | namespace {
 5 | 
 6 | int BifExample(char* path);
 7 | 
 8 | int BifExample(char* path)
 9 | {
10 |     using namespace FileFormats::Bif;
11 | 
12 |     Raw::Bif rawBif;
13 |     bool loaded = Raw::Bif::ReadFromFile(path, &rawBif);
14 | 
15 |     // Alternatively, we could have loaded the file ourselves and use 'ReadFromByteVector' or 'ReadFromBytes'.
16 | 
17 |     std::printf("BIF FileType: %.4s\n", rawBif.m_Header.m_FileType);
18 |     std::printf("BIF Version: %.4s\n", rawBif.m_Header.m_Version);
19 |     std::printf("BIF Variable resources: %u\n", rawBif.m_Header.m_VariableResourceCount);
20 |     std::printf("BIF Fixed resources: %u\n", rawBif.m_Header.m_FixedResourceCount);
21 | 
22 |     if (!loaded)
23 |     {
24 |         std::printf("Failed to load the BIF file. Check the FileType and Version and ensure the file is well formed.\n");
25 |         return 1;
26 |     }
27 | 
28 |     // Remember to use std::move here.
29 |     // If you don't, you're gonna use double the memory because everything is going to get copied.
30 |     Friendly::Bif bif(std::move(rawBif));
31 | 
32 |     std::printf("\nResources:\n");
33 | 
34 |     for (auto const& kvp : bif.GetResources())
35 |     {
36 |         // kvp.first = id
37 |         // kvp.second = Friendly::BifResource
38 |         std::printf("\n%s [%u | %u]: %zu bytes", StringFromResourceType(kvp.second.m_ResType), kvp.first, kvp.second.m_ResId, kvp.second.m_DataBlock->GetDataLength());
39 |     }
40 | 
41 |     return 0;
42 | }
43 | 
44 | }
45 | 
46 | int main(int argc, char** argv)
47 | {
48 |     ASSERT(argc == 2);
49 |     return BifExample(argv[1]);
50 | }


--------------------------------------------------------------------------------
/Examples/Example_Erf.cpp:
--------------------------------------------------------------------------------
 1 | #include "FileFormats/Erf.hpp"
 2 | #include "Utility/Assert.hpp"
 3 | 
 4 | namespace {
 5 | 
 6 | int ErfExample(char* path);
 7 | 
 8 | int ErfExample(char* path)
 9 | {
10 |     using namespace FileFormats::Erf;
11 | 
12 |     Raw::Erf rawErf;
13 |     bool loaded = Raw::Erf::ReadFromFile(path, &rawErf);
14 | 
15 |     std::printf("ERF FileType: %.4s\n", rawErf.m_Header.m_FileType);
16 |     std::printf("ERF Version: %.4s\n", rawErf.m_Header.m_Version);
17 |     std::printf("ERF Entries: %u\n", rawErf.m_Header.m_EntryCount);
18 | 
19 |     if (!loaded)
20 |     {
21 |         std::printf("Failed to load the ERF file. Check the FileType and Version and ensure the file is well formed.\n");
22 |         return 1;
23 |     }
24 | 
25 |     Friendly::Erf erf(std::move(rawErf));
26 | 
27 |     std::vector<Raw::ErfLocalisedString> const& descriptions = erf.GetDescriptions();
28 | 
29 |     if (descriptions.empty())
30 |     {
31 |         std::printf("\nNo description specified\n");
32 |     }
33 |     else
34 |     {
35 |         // Print the first description - this may not be English but probably will be.
36 |         std::printf("\nDescription: %s\n", descriptions[0].m_String.c_str());
37 |     }
38 | 
39 |     std::printf("\nResources:\n");
40 | 
41 |     for (Friendly::ErfResource const& resource : erf.GetResources())
42 |     {
43 |         const char* resType = FileFormats::Resource::StringFromResourceType(resource.m_ResType);
44 |         std::printf("\n %s.%s: %zu bytes [%u] ", resource.m_ResRef.c_str(), resType, resource.m_DataBlock->GetDataLength(), resource.m_ResourceId);
45 |     }
46 | 
47 |     return 0;
48 | }
49 | 
50 | }
51 | 
52 | int main(int argc, char** argv)
53 | {
54 |     ASSERT(argc == 2);
55 |     return ErfExample(argv[1]);
56 | }


--------------------------------------------------------------------------------
/Examples/Example_Key.cpp:
--------------------------------------------------------------------------------
 1 | #include "FileFormats/Key.hpp"
 2 | #include "Utility/Assert.hpp"
 3 | 
 4 | namespace {
 5 | 
 6 | int KeyExample(char* path);
 7 | 
 8 | int KeyExample(char* path)
 9 | {
10 |     using namespace FileFormats::Key;
11 | 
12 |     Raw::Key rawKey;
13 |     bool loaded = Raw::Key::ReadFromFile(path, &rawKey);
14 | 
15 |     std::printf("Key FileType: %.4s\n", rawKey.m_Header.m_FileType);
16 |     std::printf("Key FileVersion: %.4s\n", rawKey.m_Header.m_FileVersion);
17 |     std::printf("Key BIF Entries: %u\n", rawKey.m_Header.m_BIFCount);
18 |     std::printf("Key Entries: %u\n", rawKey.m_Header.m_KeyCount);
19 | 
20 |     if (!loaded)
21 |     {
22 |         std::printf("Failed to load the KEY file. Check the FileType and FileVersion and ensure the file is well formed.\n");
23 |         return 1;
24 |     }
25 | 
26 |     Friendly::Key key(std::move(rawKey));
27 | 
28 |     std::printf("\nReferences BIFs:\n");
29 | 
30 |     for (Friendly::KeyBifReference const& ref : key.GetReferencedBifs())
31 |     {
32 |         std::printf("\nDrives: %u, Path: %s, Size: %u", ref.m_Drives, ref.m_Path.c_str(), ref.m_FileSize);
33 |     }
34 | 
35 |     std::printf("\n\nReferenced resources:\n");
36 | 
37 |     for (Friendly::KeyBifReferencedResource const& res : key.GetReferencedResources())
38 |     {
39 |         ASSERT(res.m_ReferencedBifIndex < key.GetReferencedBifs().size());
40 |         const char* resType = FileFormats::Resource::StringFromResourceType(res.m_ResType);
41 |         std::string const& bifPath = key.GetReferencedBifs()[res.m_ReferencedBifIndex].m_Path;
42 |         std::printf("\n %s.%s %s [%zu (%u) | %u] ", res.m_ResRef.c_str(), resType, bifPath.c_str(),
43 |             res.m_ReferencedBifIndex, res.m_ReferencedBifResId, res.m_ResId);
44 |     }
45 | 
46 |     return 0;
47 | }
48 | 
49 | }
50 | 
51 | int main(int argc, char** argv)
52 | {
53 |     ASSERT(argc == 2);
54 |     return KeyExample(argv[1]);
55 | }
56 | 


--------------------------------------------------------------------------------
/Examples/Example_2da.cpp:
--------------------------------------------------------------------------------
 1 | #include "FileFormats/2da.hpp"
 2 | #include "Utility/Assert.hpp"
 3 | 
 4 | namespace {
 5 | 
 6 | int TwoDAExample(char* path);
 7 | 
 8 | int TwoDAExample(char* path)
 9 | {
10 |     using namespace FileFormats::TwoDA;
11 | 
12 |     Raw::TwoDA raw2da;
13 |     bool loaded = Raw::TwoDA::ReadFromFile(path, &raw2da);
14 | 
15 |     std::printf("2DA Lines: %zu\n", raw2da.m_Lines.size());
16 | 
17 |     if (!loaded)
18 |     {
19 |         std::printf("Failed to load the 2DA file.\n");
20 |         return 1;
21 |     }
22 | 
23 |     Friendly::TwoDA twoDA(std::move(raw2da));
24 | 
25 |     std::vector<std::string> columnNames;
26 | 
27 |     // Convert column names from map to a flat vector.
28 |     for (auto& kvp : twoDA.GetColumnNames())
29 |     {
30 |         if (columnNames.size() < kvp.second + 1)
31 |         {
32 |             columnNames.resize(kvp.second + 1);
33 |         }
34 |         columnNames[kvp.second] = kvp.first;
35 |     }
36 | 
37 |     std::printf("\n");
38 | 
39 |     for (std::string const& column : columnNames)
40 |     {
41 |         std::printf("%-16s ", column.c_str());
42 |     }
43 | 
44 |     std::printf("\n\n");
45 | 
46 |     for (Friendly::TwoDARow const& row : twoDA)
47 |     {
48 |         for (std::size_t i = 0; i < row.Size(); ++i)
49 |         {
50 |             Friendly::TwoDAEntry const& entry = row[i];
51 | 
52 |             // Copy here so we can modify it later.
53 |             std::string str = entry.m_IsEmpty ? "[MISSING_DATA]" : entry.m_Data;
54 | 
55 |             if (!entry.m_IsEmpty && str.find(' ') != std::string::npos)
56 |             {
57 |                 // We have whitespace, so we should surround the entry with quotes when pretty printing it.
58 |                 str = "\"" + str + "\"";
59 |             }
60 | 
61 |             std::printf("%-16s ", str.c_str());
62 |         }
63 | 
64 |         std::printf("\n");
65 |     }
66 | 
67 |     // Then save the 2da back out next to the original.
68 |     char pathBuffer[1024];
69 |     std::sprintf(pathBuffer, "%s.1", path);
70 |     bool written = twoDA.WriteToFile(pathBuffer);
71 |     ASSERT(written);
72 | 
73 |     return 0;
74 | }
75 | 
76 | }
77 | 
78 | int main(int argc, char** argv)
79 | {
80 |     ASSERT(argc == 2);
81 |     return TwoDAExample(argv[1]);
82 | }
83 | 


--------------------------------------------------------------------------------
/FileFormats/Key/Key_Friendly.cpp:
--------------------------------------------------------------------------------
 1 | #include "FileFormats/Key/Key_Friendly.hpp"
 2 | #include "Utility/Assert.hpp"
 3 | 
 4 | #include <algorithm>
 5 | #include <cstring>
 6 | 
 7 | namespace FileFormats::Key::Friendly {
 8 | 
 9 | Key::Key(Raw::Key const& rawKey)
10 | {
11 |     // Get the referenced BIFs.
12 |     for (Raw::KeyFile const& rawFile : rawKey.m_Files)
13 |     {
14 |         KeyBifReference reference;
15 |         reference.m_Drives = rawFile.m_Drives;
16 |         reference.m_FileSize = rawFile.m_FileSize;
17 | 
18 |         std::uint32_t offSetStartIntoFilenameTable = rawKey.m_Header.m_OffsetToFileTable + (rawKey.m_Header.m_BIFCount * sizeof(Raw::KeyFile)); // End of file table
19 |         std::uint32_t offSetIntoFilenameTable = rawFile.m_FilenameOffset - offSetStartIntoFilenameTable;
20 |         ASSERT(offSetStartIntoFilenameTable + offSetIntoFilenameTable + rawFile.m_FilenameSize <= rawKey.m_Header.m_OffsetToKeyTable);
21 | 
22 |         char const* ptr = rawKey.m_Filenames.data() + offSetIntoFilenameTable;
23 |         reference.m_Path = std::string(ptr, strnlen(ptr, rawFile.m_FilenameSize));
24 | 
25 |         // Replace all back slash with forward slashes. This avoids any nasty platform-related issues.
26 |         std::replace(std::begin(reference.m_Path), std::end(reference.m_Path), '\\', '/');
27 | 
28 |         m_ReferencedBifs.emplace_back(std::move(reference));
29 |     }
30 | 
31 |     // Get the references resources.
32 |     for (Raw::KeyEntry const& rawEntry : rawKey.m_Entries)
33 |     {
34 |         std::string resref = std::string(rawEntry.m_ResRef, rawEntry.m_ResRef + strnlen(rawEntry.m_ResRef, 16));
35 | 
36 |         // NWN is case insensitive and cases are mixed like crazy in the official modules.
37 |         // We just do the conversion to lower here to simplify things.
38 |         std::transform(std::begin(resref), std::end(resref), std::begin(resref), ::tolower);
39 | 
40 |         KeyBifReferencedResource entry;
41 |         entry.m_ResRef = std::move(resref);
42 |         entry.m_ResType = rawEntry.m_ResourceType;
43 |         entry.m_ResId = rawEntry.m_ResID;
44 |         entry.m_ReferencedBifResId = rawEntry.m_ResID & 0x00003FFF; // See Bif_Friendly.cpp for explanation.
45 |         entry.m_ReferencedBifIndex = rawEntry.m_ResID >> 20;
46 | 
47 |         m_ReferencedResources.emplace_back(std::move(entry));
48 |     }
49 | }
50 | 
51 | std::vector<KeyBifReference> const& Key::GetReferencedBifs() const
52 | {
53 |     return m_ReferencedBifs;
54 | }
55 | 
56 | std::vector<KeyBifReferencedResource> const& Key::GetReferencedResources() const
57 | {
58 |     return m_ReferencedResources;
59 | }
60 | 
61 | }
62 | 


--------------------------------------------------------------------------------
/Utility/MemoryMappedFile_impl.cpp:
--------------------------------------------------------------------------------
  1 | #include "MemoryMappedFile_impl.hpp"
  2 | #include "Utility/Assert.hpp"
  3 | 
  4 | #if OS_LINUX
  5 |     #include <fcntl.h>
  6 |     #include <sys/mman.h>
  7 |     #include <sys/stat.h>
  8 |     #include <unistd.h>
  9 | #endif
 10 | 
 11 | MemoryMappedFile_impl::MemoryMappedFile_impl()
 12 | #if OS_WINDOWS
 13 |     : m_File(INVALID_HANDLE_VALUE), m_MemoryMap(INVALID_HANDLE_VALUE), m_Ptr(NULL)
 14 | #else
 15 |     : m_FileDescriptor(-1), m_Ptr(nullptr), m_PtrLength(0)
 16 | #endif
 17 | { }
 18 | 
 19 | MemoryMappedFile_impl::~MemoryMappedFile_impl()
 20 | {
 21 | #if OS_WINDOWS
 22 |     if (m_Ptr != NULL)
 23 |     {
 24 |         UnmapViewOfFile(m_Ptr);
 25 |     }
 26 | 
 27 |     if (m_MemoryMap != INVALID_HANDLE_VALUE)
 28 |     {
 29 |         CloseHandle(m_MemoryMap);
 30 |     }
 31 | 
 32 |     if (m_File != INVALID_HANDLE_VALUE)
 33 |     {
 34 |         CloseHandle(m_File);
 35 |     }
 36 | #else
 37 |     if (m_Ptr == MAP_FAILED)
 38 |     {
 39 |         munmap(m_Ptr, m_PtrLength);
 40 |     }
 41 | 
 42 |     if (m_FileDescriptor != -1)
 43 |     {
 44 |         close(m_FileDescriptor);
 45 |     }
 46 | #endif
 47 | }
 48 | 
 49 | bool MemoryMappedFile_impl::Map(char const* path, NonOwningDataBlock* out)
 50 | {
 51 |     ASSERT(path);
 52 |     ASSERT(out);
 53 | 
 54 | #if OS_WINDOWS
 55 |     m_File = CreateFile(path, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
 56 |     if (m_File == INVALID_HANDLE_VALUE)
 57 |     {
 58 |         return false;
 59 |     }
 60 | 
 61 |     m_MemoryMap = CreateFileMapping(m_File, NULL, PAGE_READONLY, 0, 0, NULL);
 62 |     if (m_MemoryMap == INVALID_HANDLE_VALUE)
 63 |     {
 64 |         return false;
 65 |     }
 66 | 
 67 |     m_Ptr = MapViewOfFile(m_MemoryMap, FILE_MAP_READ, 0, 0, 0);
 68 |     if (!m_Ptr)
 69 |     {
 70 |         return false;
 71 |     }
 72 | 
 73 |     out->m_Data = static_cast<std::byte*>(m_Ptr);
 74 |     out->m_DataLength = GetFileSize(m_File, NULL);
 75 | #else
 76 |     m_FileDescriptor = open(path, O_RDONLY);
 77 |     if (m_FileDescriptor == -1)
 78 |     {
 79 |         return false;
 80 |     }
 81 | 
 82 |     struct stat statBuffer;
 83 |     if (stat(path, &statBuffer) == -1)
 84 |     {
 85 |         return false;
 86 |     }
 87 | 
 88 |     m_PtrLength = statBuffer.st_size;
 89 |     m_Ptr = mmap(nullptr, m_PtrLength, PROT_READ, MAP_PRIVATE, m_FileDescriptor, 0);
 90 | 
 91 |     if (m_Ptr == MAP_FAILED)
 92 |     {
 93 |         return false;
 94 |     }
 95 | 
 96 |     out->m_Data = static_cast<std::byte*>(m_Ptr);
 97 |     out->m_DataLength = m_PtrLength;
 98 | #endif
 99 | 
100 |     return true;
101 | }
102 | 


--------------------------------------------------------------------------------
/Tools/Tool_2daMerge.cpp:
--------------------------------------------------------------------------------
 1 | #include "FileFormats/2da.hpp"
 2 | #include "Utility/Assert.hpp"
 3 | 
 4 | namespace {
 5 | 
 6 | int TwoDAMerge(char* base, char* other, char* out)
 7 | {
 8 |     using namespace FileFormats::TwoDA;
 9 | 
10 |     Raw::TwoDA baseTwoDARaw;
11 | 
12 |     if (!Raw::TwoDA::ReadFromFile(base, &baseTwoDARaw))
13 |     {
14 |         std::printf("Failed to load base 2da from %s.\n", base);
15 |         return 1;
16 |     }
17 | 
18 |     Friendly::TwoDA baseTwoDA(std::move(baseTwoDARaw));
19 | 
20 |     // Iterate over the base 2da and warn if any rows are misnumbered.
21 | 
22 |     {
23 |         std::uint32_t i = 0;
24 |         for (const Friendly::TwoDARow& row : baseTwoDA)
25 |         {
26 |             if (row.RowId() != i++)
27 |             {
28 |                 std::printf("Warning: Row %u with ID %u - be careful - the row IDs may be off in this file!\n", i, row.RowId());
29 |                 break;
30 |             }
31 |         }
32 |     }
33 | 
34 |     Raw::TwoDA otherTwoDARaw;
35 | 
36 |     if (!Raw::TwoDA::ReadFromFile(other, &otherTwoDARaw))
37 |     {
38 |         std::printf("Failed to load other 2da from %s.\n", other);
39 |         return 1;
40 |     }
41 | 
42 |     Friendly::TwoDA otherTwoDA(std::move(otherTwoDARaw));
43 | 
44 |     ASSERT(baseTwoDA.GetColumnNames().size() == otherTwoDA.GetColumnNames().size());
45 | 
46 |     std::vector<const Friendly::TwoDARow*> needToMerge;
47 | 
48 |     // Write over anything we've changed.
49 |     for (const Friendly::TwoDARow& row : otherTwoDA)
50 |     {
51 |         std::uint32_t rowId = row.RowId();
52 |         if (static_cast<int>(rowId) < std::end(baseTwoDA) - std::begin(baseTwoDA))
53 |         {
54 |             auto iterIntoBase = std::begin(baseTwoDA) + rowId;
55 |             for (std::size_t i = 0; i < iterIntoBase->Size(); ++i)
56 |             {
57 |                 (*iterIntoBase)[i] = row[i];
58 |             }
59 |         }
60 |         else
61 |         {
62 |             needToMerge.emplace_back(&row);
63 |         }
64 |     }
65 | 
66 |     // Insert anything we've added.
67 |     for (const Friendly::TwoDARow* row : needToMerge)
68 |     {
69 |         Friendly::TwoDARow& baseRow = baseTwoDA[row->RowId()];
70 |         for (std::size_t i = 0; i < row->Size(); ++i)
71 |         {
72 |             baseRow[i] = (*row)[i];
73 |         }
74 |     }
75 | 
76 |     // Then save the 2da back out next to the original.
77 |     if (!baseTwoDA.WriteToFile(out))
78 |     {
79 |         std::printf("Failed to save merged 2da to %s.\n", out);
80 |         return 1;
81 |     }
82 | 
83 |     return 0;
84 | }
85 | 
86 | }
87 | 
88 | int main(int argc, char** argv)
89 | {
90 |     if (argc != 4)
91 |     {
92 |         std::printf("2da_merge [base2dapath] [other2dapath] [out2dapath]\n");
93 |         return 1;
94 |     }
95 | 
96 |     return TwoDAMerge(argv[1], argv[2], argv[3]);
97 | }
98 | 


--------------------------------------------------------------------------------
/Utility/Assert.cpp:
--------------------------------------------------------------------------------
  1 | #include "Utility/Assert.hpp"
  2 | 
  3 | #include <cstdio>
  4 | #include <cstdlib>
  5 | #include <cstring>
  6 | 
  7 | #if OS_WINDOWS
  8 |     #include "Windows.h"
  9 | #elif OS_LINUX
 10 |     #include <execinfo.h>
 11 |     #include <signal.h>
 12 | #endif
 13 | 
 14 | namespace Assert {
 15 | 
 16 | void Fail(const char* condition, const char* file, int line, const char* message)
 17 | {
 18 |     char buffer[2048];
 19 | 
 20 |     if (condition)
 21 |     {
 22 |         std::sprintf(buffer, "ASSERTION FAILURE\n  Summary: (%s) failed at (%s:%d)", condition, file, line);
 23 |     }
 24 |     else
 25 |     {
 26 |         std::sprintf(buffer, "ASSERTION FAILURE\n  Summary: Failed at (%s:%d)", file, line);
 27 |     }
 28 | 
 29 |     if (message)
 30 |     {
 31 |         std::strcat(buffer, "\n  Message: ");
 32 |         std::strcat(buffer, message);
 33 |     }
 34 | 
 35 | #if OS_WINDOWS
 36 |     void* stackTrace[20];
 37 |     int numCapturedFrames = CaptureStackBackTrace(0, 20, stackTrace, NULL);
 38 | 
 39 |     if (numCapturedFrames)
 40 |     {
 41 |         std::strcat(buffer, "\n  Backtrace:\n");
 42 |         for (int i = 0; i < numCapturedFrames; ++i)
 43 |         {
 44 |             char backtraceBuffer[32];
 45 |             std::sprintf(backtraceBuffer, "    [0x%p]\n", stackTrace[i]);
 46 |             std::strcat(buffer, backtraceBuffer);
 47 |         }
 48 |     }
 49 | #elif OS_LINUX
 50 |     void* stackTrace[20];
 51 |     int numCapturedFrames = backtrace(stackTrace, 20);
 52 | 
 53 |     if (numCapturedFrames)
 54 |     {
 55 |         char** resolvedFrames = backtrace_symbols(stackTrace, 20);
 56 |         std::strcat(buffer, "\n  Backtrace:\n");
 57 |         for (int i = 0; i < numCapturedFrames; ++i)
 58 |         {
 59 |             char backtraceBuffer[256];
 60 |             std::sprintf(backtraceBuffer, "    %s\n", resolvedFrames[i]);
 61 |             std::strcat(buffer, backtraceBuffer);
 62 |         }
 63 |     }
 64 | #endif // OS_WINDOWS
 65 | 
 66 |     std::fprintf(stderr, "%s", buffer);
 67 | 
 68 |     bool skipCrash = false;
 69 |     bool skipBreak = false;
 70 | 
 71 | #if OS_WINDOWS
 72 |     int response = MessageBox(GetActiveWindow(), buffer, "ASSERTION FAILURE", MB_ABORTRETRYIGNORE);
 73 | 
 74 |     switch (response)
 75 |     {
 76 |     case IDRETRY: // No crash, but break.
 77 |         skipCrash = true;
 78 |         break;
 79 | 
 80 |     case IDIGNORE: // No crash or break.
 81 |         skipCrash = true;
 82 |         skipBreak = true;
 83 |         break;
 84 | 
 85 |     case IDABORT:
 86 |     default: // Crash and break
 87 |         break;
 88 |     }
 89 | #endif // OS_WINDOWS
 90 | 
 91 |     if (!skipBreak)
 92 |     {
 93 | #if CMP_MSVC
 94 |         __debugbreak();
 95 | #elif OS_LINUX
 96 |         raise(SIGTRAP);
 97 | #endif
 98 |     }
 99 | 
100 |     if (!skipCrash)
101 |     {
102 |         std::abort();
103 |     }
104 | }
105 | 
106 | }
107 | 


--------------------------------------------------------------------------------
/FileFormats/Tlk/Tlk_Raw.cpp:
--------------------------------------------------------------------------------
  1 | #include "FileFormats/Tlk/Tlk_Raw.hpp"
  2 | #include "Utility/Assert.hpp"
  3 | #include "Utility/MemoryMappedFile.hpp"
  4 | 
  5 | #include <cstring>
  6 | 
  7 | namespace {
  8 | 
  9 | template <typename T>
 10 | void ReadGenericOffsetable(std::byte const* bytesWithInitialOffset, std::size_t count, std::vector<T>& out)
 11 | {
 12 |     out.resize(count);
 13 |     std::memcpy(out.data(), bytesWithInitialOffset, count * sizeof(T));
 14 | }
 15 | 
 16 | }
 17 | 
 18 | namespace FileFormats::Tlk::Raw {
 19 | 
 20 | bool Tlk::ReadFromBytes(std::byte const* bytes, std::size_t bytesCount, Tlk* out)
 21 | {
 22 |     ASSERT(bytes);
 23 |     ASSERT(out);
 24 |     return out->ConstructInternal(bytes, bytesCount);
 25 | }
 26 | 
 27 | bool Tlk::ReadFromByteVector(std::vector<std::byte>&& bytes, Tlk* out)
 28 | {
 29 |     ASSERT(!bytes.empty());
 30 |     ASSERT(out);
 31 |     return out->ConstructInternal(bytes.data(), bytes.size());
 32 | }
 33 | 
 34 | bool Tlk::ReadFromFile(char const* path, Tlk* out)
 35 | {
 36 |     ASSERT(path);
 37 |     ASSERT(out);
 38 | 
 39 |     MemoryMappedFile memmap;
 40 |     bool loaded = MemoryMappedFile::MemoryMap(path, &memmap);
 41 | 
 42 |     if (!loaded)
 43 |     {
 44 |         return false;
 45 |     } 
 46 | 
 47 |     return out->ConstructInternal(memmap.GetDataBlock().GetData(), memmap.GetDataBlock().GetDataLength());
 48 | }
 49 | 
 50 | bool Tlk::WriteToFile(char const* path) const
 51 | {
 52 |     ASSERT(path);
 53 | 
 54 |     FILE* outFile = std::fopen(path, "wb");
 55 | 
 56 |     if (outFile)
 57 |     {
 58 |         std::fwrite(&m_Header, sizeof(m_Header), 1, outFile);
 59 |         std::fwrite(m_StringData.data(), sizeof(m_StringData[0]), m_StringData.size(), outFile);
 60 |         std::fwrite(m_StringEntries.data(), sizeof(m_StringEntries[0]), m_StringEntries.size(), outFile);
 61 |         std::fclose(outFile);
 62 |         return true;
 63 |     }
 64 | 
 65 |     return false;
 66 | }
 67 | 
 68 | bool Tlk::ConstructInternal(std::byte const* bytes, std::size_t bytesCount)
 69 | {
 70 |     ASSERT(bytes);
 71 | 
 72 |     std::memcpy(&m_Header, bytes, sizeof(m_Header));
 73 | 
 74 |     if (std::memcmp(m_Header.m_FileType, "TLK ", 4) != 0 ||
 75 |         std::memcmp(m_Header.m_FileVersion, "V3.0", 4) != 0)
 76 |     {
 77 |         return false;
 78 |     }
 79 | 
 80 |     ReadStringData(bytes);
 81 |     ReadStringEntries(bytes, bytesCount);
 82 | 
 83 |     return true;
 84 | }
 85 | 
 86 | void Tlk::ReadStringData(std::byte const* data)
 87 | {
 88 |     std::uint32_t offset = sizeof(m_Header);
 89 |     std::uint32_t count = m_Header.m_StringCount;
 90 |     ReadGenericOffsetable(data + offset, count, m_StringData);
 91 | }
 92 | 
 93 | void Tlk::ReadStringEntries(std::byte const* data, std::size_t bytesCount)
 94 | {
 95 |     std::uint32_t offset = m_Header.m_StringEntriesOffset;
 96 |     std::size_t count = bytesCount - offset;
 97 |     ReadGenericOffsetable(data + offset, count, m_StringEntries);
 98 | }
 99 | 
100 | }
101 | 


--------------------------------------------------------------------------------
/FileFormats/Bif/Bif_Friendly.cpp:
--------------------------------------------------------------------------------
 1 | #include "FileFormats/Bif/Bif_Friendly.hpp"
 2 | #include "Utility/Assert.hpp"
 3 | #include "Utility/DataBlock.hpp"
 4 | 
 5 | #include <cstring>
 6 | 
 7 | namespace FileFormats::Bif::Friendly {
 8 | 
 9 | Bif::Bif(Raw::Bif const& rawBif)
10 | {
11 |     ConstructInternal(rawBif);
12 | }
13 | 
14 | Bif::Bif(Raw::Bif&& rawBif) : m_RawBif(std::forward<Raw::Bif>(rawBif))
15 | {
16 |     ConstructInternal(m_RawBif.value());
17 | }
18 | 
19 | void Bif::ConstructInternal(Raw::Bif const& rawBif)
20 | {
21 |     ASSERT(!rawBif.m_Header.m_FixedResourceCount);
22 | 
23 |     // Calculate the offset into the data block manually ...
24 |     std::size_t offsetToDataBlock = rawBif.m_Header.m_VariableTableOffset;
25 |     offsetToDataBlock += rawBif.m_VariableResourceTable.size() * sizeof(Raw::BifVariableResource);
26 |     offsetToDataBlock += rawBif.m_FixedResourceTable.size() * sizeof(Raw::BifFixedResource);
27 | 
28 |     for (Raw::BifVariableResource const& rawRes : rawBif.m_VariableResourceTable)
29 |     {
30 |         ASSERT(m_Resources.find(rawRes.m_Id) == std::end(m_Resources));
31 | 
32 |         BifResource res;
33 | 
34 |         res.m_ResId = rawRes.m_Id;
35 |         res.m_ResType = rawRes.m_ResourceType;
36 | 
37 |         std::size_t offsetToData = rawRes.m_Offset - offsetToDataBlock;
38 |         ASSERT(offsetToData + rawRes.m_FileSize <= rawBif.m_DataBlock->GetDataLength());
39 | 
40 |         if (m_RawBif.has_value())
41 |         {
42 |             std::unique_ptr<NonOwningDataBlock> db = std::make_unique<NonOwningDataBlock>();
43 |             db->m_Data = rawBif.m_DataBlock->GetData() + offsetToData;
44 |             db->m_DataLength = rawRes.m_FileSize;
45 |             res.m_DataBlock = std::move(db);
46 |         }
47 |         else
48 |         {
49 |             std::unique_ptr<OwningDataBlock> db = std::make_unique<OwningDataBlock>();
50 |             db->m_Data.resize(rawRes.m_FileSize);
51 |             std::memcpy(db->m_Data.data(), rawBif.m_DataBlock->GetData() + offsetToData, rawRes.m_FileSize);
52 |             res.m_DataBlock = std::move(db);
53 |         }
54 | 
55 |         // The spec outlines this the m_ReferencedBifResId as (x << 20) + y, where y is the index, and x = y normally and 0
56 |         // for patch BIFs. However, none of the BIFs present in 1.69 or 1.74 seem to follow this rule - x always equals y.
57 |         //
58 |         // There exists no entry that specifies x. In the spec, it also states that the game doesn't care about the mismatch
59 |         // between x and y.
60 |         //
61 |         // Therefore, I'm just going to do what the game does - mask out anything higher than 0x00003FFF
62 |         // (bottom fourteen bits) so we can just completely ignore the x value.
63 |         //
64 |         // We'll use this as the index into the BIF while maintaining the original ID as an entry field.
65 | 
66 |         std::uint32_t keyAddressableId = rawRes.m_Id & 0x00003FFF;
67 | 
68 |         m_Resources.insert(std::make_pair(keyAddressableId, std::move(res)));
69 |     }
70 | }
71 | 
72 | Bif::BifResourceMap const& Bif::GetResources() const
73 | {
74 |     return m_Resources;
75 | }
76 | 
77 | }
78 | 


--------------------------------------------------------------------------------
/FileFormats/Erf/Erf_Friendly.cpp:
--------------------------------------------------------------------------------
 1 | #include "FileFormats/Erf/Erf_Friendly.hpp"
 2 | #include "Utility/Assert.hpp"
 3 | 
 4 | #include <algorithm>
 5 | #include <cstring>
 6 | 
 7 | namespace FileFormats::Erf::Friendly {
 8 | 
 9 | Erf::Erf(Raw::Erf const& rawErf)
10 | {
11 |     ConstructInternal(rawErf);
12 | }
13 | 
14 | Erf::Erf(Raw::Erf&& rawErf) : m_RawErf(std::forward<Raw::Erf>(rawErf))
15 | {
16 |     ConstructInternal(m_RawErf.value());
17 | }
18 | 
19 | std::vector<Raw::ErfLocalisedString> const& Erf::GetDescriptions() const
20 | {
21 |     return m_Descriptions;
22 | }
23 | 
24 | std::vector<ErfResource> const& Erf::GetResources() const
25 | {
26 |     return m_Resources;
27 | }
28 | 
29 | void Erf::ConstructInternal(Raw::Erf const& rawErf)
30 | {
31 |     // First - copy in the descriptions. This one is simple.
32 |     m_Descriptions = rawErf.m_LocalisedStrings;
33 | 
34 |     ASSERT(rawErf.m_Resources.size() == rawErf.m_Header.m_EntryCount);
35 |     ASSERT(rawErf.m_Keys.size() == rawErf.m_Header.m_EntryCount);
36 | 
37 |     // Second - iterate over every entry, then set them up in a user friendly way.
38 |     for (std::size_t i = 0; i < rawErf.m_Header.m_EntryCount; ++i)
39 |     {
40 |         Raw::ErfKey const& rawKey = rawErf.m_Keys[i];
41 |         Raw::ErfResource const& rawRes = rawErf.m_Resources[i];
42 | 
43 |         std::string resref = std::string(rawKey.m_ResRef, rawKey.m_ResRef + strnlen(rawKey.m_ResRef, 16));
44 | 
45 |         // NWN is case insensitive and cases are mixed like crazy in the official modules.
46 |         // We just do the conversion to lower here to simplify things.
47 |         std::transform(std::begin(resref), std::end(resref), std::begin(resref), ::tolower);
48 | 
49 |         ErfResource resource;
50 |         resource.m_ResRef = std::move(resref);
51 |         resource.m_ResType = rawKey.m_ResType;
52 |         resource.m_ResourceId = rawKey.m_ResId;
53 | 
54 |         // Per the spec, the resourceID should match exactly the order that the resources are present in the resource block.
55 |         // We assert here to ensure that is actually the case.
56 |         ASSERT(resource.m_ResourceId == i);
57 | 
58 |         // This gives us the offset to the start of the resource data block.
59 |         std::uint32_t offsetToEndOfResources = rawErf.m_Header.m_OffsetToResourceList + (sizeof(Raw::ErfResource) * rawErf.m_Header.m_EntryCount); // End of resources block
60 |         std::size_t offsetIntoResourceData = rawRes.m_OffsetToResource - offsetToEndOfResources;
61 |         ASSERT(offsetIntoResourceData < rawErf.m_ResourceData->GetDataLength());
62 | 
63 |         if (m_RawErf.has_value())
64 |         {
65 |             std::unique_ptr<NonOwningDataBlock> db = std::make_unique<NonOwningDataBlock>();
66 |             db->m_Data = rawErf.m_ResourceData->GetData() + offsetIntoResourceData;
67 |             db->m_DataLength = rawRes.m_ResourceSize;
68 |             resource.m_DataBlock = std::move(db);
69 |         }
70 |         else
71 |         {
72 |             std::unique_ptr<OwningDataBlock> db = std::make_unique<OwningDataBlock>();
73 |             db->m_Data.resize(rawRes.m_ResourceSize);
74 |             std::memcpy(db->m_Data.data(), rawErf.m_ResourceData->GetData() + offsetIntoResourceData, rawRes.m_ResourceSize);
75 |             resource.m_DataBlock = std::move(db);
76 |         }
77 | 
78 |         m_Resources.emplace_back(std::move(resource));
79 |     }
80 | }
81 | 
82 | }
83 | 


--------------------------------------------------------------------------------
/FileFormats/2da/2da_Friendly.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include "FileFormats/2da/2da_Raw.hpp"
 4 | 
 5 | #include <unordered_map>
 6 | 
 7 | namespace FileFormats::TwoDA::Friendly {
 8 | 
 9 | struct TwoDAEntry
10 | {
11 |     // The data contained in this entry.
12 |     std::string m_Data;
13 | 
14 |     // This entry is an empty value.
15 |     bool m_IsEmpty;
16 | };
17 | 
18 | class TwoDARow
19 | {
20 | public:
21 |     TwoDARow(std::uint32_t rowId,
22 |         std::vector<TwoDAEntry>&& data,
23 |         std::unordered_map<std::string, std::size_t> const& columns);
24 | 
25 |     // Operator[] returns the column directly.
26 |     // Out-of-range access is not supported at this time.
27 |     TwoDAEntry& operator[](std::size_t column);
28 |     TwoDAEntry& operator[](std::string const& column);
29 |     TwoDAEntry const& operator[](std::size_t column) const;
30 |     TwoDAEntry const& operator[](std::string const& column) const;
31 | 
32 |     // These functions can be used to extract the value as the specified type.
33 |     std::string const& AsStr(std::size_t column) const;
34 |     std::string const& AsStr(std::string const& column) const;
35 | 
36 |     std::int32_t AsInt(std::size_t column) const;
37 |     std::int32_t AsInt(std::string const& column) const;
38 | 
39 |     float AsFloat(std::size_t column) const;
40 |     float AsFloat(std::string const& column) const;
41 | 
42 |     bool IsEmpty(std::size_t column) const;
43 |     bool IsEmpty(std::string const& column) const;
44 | 
45 |     std::uint32_t RowId() const;
46 | 
47 |     using TwoDAEntries = std::vector<TwoDAEntry>;
48 |     TwoDAEntries::iterator begin();
49 |     TwoDAEntries::iterator end();
50 |     TwoDAEntries::const_iterator begin() const;
51 |     TwoDAEntries::const_iterator end() const;
52 |     std::size_t Size() const;
53 | 
54 | private:
55 |     std::uint32_t m_RowId;
56 |     std::unordered_map<std::string, std::size_t> const& m_ColumnNames;
57 |     std::vector<TwoDAEntry> m_Data;
58 | };
59 | 
60 | class TwoDA
61 | {
62 | public:
63 |     TwoDA(Raw::TwoDA const& raw2da);
64 | 
65 |     // These functions can be used to extract the value as the specified type.
66 |     std::string const& AsStr(std::size_t row, std::size_t column) const;
67 |     std::string const& AsStr(std::size_t row, std::string const& column) const;
68 | 
69 |     std::int32_t AsInt(std::size_t row, std::size_t column) const;
70 |     std::int32_t AsInt(std::size_t row, std::string const& column) const;
71 | 
72 |     float AsFloat(std::size_t row, std::size_t column) const;
73 |     float AsFloat(std::size_t row, std::string const& column) const;
74 | 
75 |     // Operator[] returns the row directly.
76 |     // Inserts a row if they do not exist - also fills in preceeding rows if they do not exist.
77 |     TwoDARow& operator[](std::size_t row);
78 | 
79 |     // This is just a flat vector of rows.
80 |     using TwoDARows = std::vector<TwoDARow>;
81 |     TwoDARows::iterator begin();
82 |     TwoDARows::iterator end();
83 |     TwoDARows::const_iterator begin() const;
84 |     TwoDARows::const_iterator end() const;
85 |     std::size_t Size() const;
86 | 
87 |     // The column map is a map, where the index contains the name of the column.
88 |     std::unordered_map<std::string, std::size_t> const& GetColumnNames() const;
89 | 
90 |     bool WriteToFile(char const* path) const;
91 | 
92 | private:
93 |     TwoDARows m_Rows;
94 |     std::unordered_map<std::string, std::size_t> m_ColumnNames;
95 | };
96 | 
97 | }
98 | 


--------------------------------------------------------------------------------
/FileFormats/Key/Key_Raw.cpp:
--------------------------------------------------------------------------------
  1 | #include "FileFormats/Key/Key_Raw.hpp"
  2 | #include "Utility/Assert.hpp"
  3 | #include "Utility/MemoryMappedFile.hpp"
  4 | 
  5 | #include <cstring>
  6 | 
  7 | namespace {
  8 | 
  9 | template <typename T>
 10 | void ReadGenericOffsetable(std::byte const* bytesWithInitialOffset, std::size_t count, std::vector<T>& out)
 11 | {
 12 |     out.resize(count);
 13 |     std::memcpy(out.data(), bytesWithInitialOffset, count * sizeof(T));
 14 | }
 15 | 
 16 | }
 17 | 
 18 | namespace FileFormats::Key::Raw {
 19 | 
 20 | bool Key::ReadFromBytes(std::byte const* bytes, Key* out)
 21 | {
 22 |     ASSERT(bytes);
 23 |     ASSERT(out);
 24 |     return out->ConstructInternal(bytes);
 25 | }
 26 | 
 27 | bool Key::ReadFromByteVector(std::vector<std::byte>&& bytes, Key* out)
 28 | {
 29 |     ASSERT(!bytes.empty());
 30 |     ASSERT(out);
 31 |     return out->ConstructInternal(bytes.data());
 32 | }
 33 | 
 34 | bool Key::ReadFromFile(char const* path, Key* out)
 35 | {
 36 |     ASSERT(path);
 37 |     ASSERT(out);
 38 | 
 39 |     MemoryMappedFile memmap;
 40 |     bool loaded = MemoryMappedFile::MemoryMap(path, &memmap);
 41 | 
 42 |     if (!loaded)
 43 |     {
 44 |         return false;
 45 |     }
 46 | 
 47 |     return out->ConstructInternal(memmap.GetDataBlock().GetData());
 48 | }
 49 | 
 50 | bool Key::ConstructInternal(std::byte const* bytes)
 51 | {
 52 |     ASSERT(bytes);
 53 | 
 54 |     std::memcpy(&m_Header, bytes, sizeof(m_Header));
 55 | 
 56 |     if (std::memcmp(m_Header.m_FileType, "KEY ", 4) != 0 ||
 57 |         std::memcmp(m_Header.m_FileVersion, "V1  ", 4) != 0)
 58 |     {
 59 |         return false;
 60 |     }
 61 | 
 62 |     ReadFiles(bytes);
 63 |     ReadFilenames(bytes);
 64 |     ReadEntries(bytes);
 65 | 
 66 |     return true;
 67 | }
 68 | 
 69 | void Key::ReadFiles(std::byte const* data)
 70 | {
 71 |     std::uint32_t offset = m_Header.m_OffsetToFileTable;
 72 |     std::uint32_t count = m_Header.m_BIFCount;
 73 |     ReadGenericOffsetable(data + offset, count, m_Files);
 74 | }
 75 | 
 76 | void Key::ReadFilenames(std::byte const* data)
 77 | {
 78 |     std::uint32_t offset = m_Header.m_OffsetToFileTable + (m_Header.m_BIFCount * sizeof(KeyFile)); // End of file table
 79 |     std::uint32_t count = m_Header.m_OffsetToKeyTable - offset; // Between the file table and the key table.
 80 |     ReadGenericOffsetable(data + offset, count, m_Filenames);
 81 | }
 82 | 
 83 | void Key::ReadEntries(std::byte const* data)
 84 | {
 85 |     std::uint32_t offset = m_Header.m_OffsetToKeyTable;
 86 |     std::uint32_t count = m_Header.m_KeyCount;
 87 | 
 88 |     data = data + offset;
 89 | 
 90 |     for (std::size_t i = 0; i < count; ++i)
 91 |     {
 92 |         // Because of structure padding we need to serialise this manually.
 93 |         // We could avoid this by packing the structure but there doesn't seem to be a modern C++ standard 
 94 |         // way of doing this and I don't want to whip out the pragma pack macros for this.
 95 |         // We can change this if this operation becomes too slow.
 96 | 
 97 |         KeyEntry entry;
 98 | 
 99 |         std::memcpy(&entry.m_ResRef, data, sizeof(entry.m_ResRef));
100 |         data += sizeof(entry.m_ResRef);
101 | 
102 |         std::memcpy(&entry.m_ResourceType, data, sizeof(entry.m_ResourceType));
103 |         data += sizeof(entry.m_ResourceType);
104 | 
105 |         std::memcpy(&entry.m_ResID, data, sizeof(entry.m_ResID));
106 |         data += sizeof(entry.m_ResID);
107 | 
108 |         m_Entries.emplace_back(std::move(entry));
109 |     }
110 | }
111 | 
112 | }
113 | 


--------------------------------------------------------------------------------
/CMakeLists.txt:
--------------------------------------------------------------------------------
  1 | cmake_minimum_required(VERSION 3.0.2)
  2 | project(NWNFileFormats)
  3 | 
  4 | # Platform and compiler defines
  5 | 
  6 | if(${CMAKE_CXX_COMPILER_ID} STREQUAL MSVC)
  7 |     set(CMP_MSVC 1)
  8 | elseif(${CMAKE_CXX_COMPILER_ID} STREQUAL Clang)
  9 |     set(CMP_CLANG 1)
 10 | elseif(${CMAKE_CXX_COMPILER_ID} STREQUAL Intel)
 11 |     set(CMP_INTEL 1)
 12 | else() # If we can't detect a compiler, we presume GCC.
 13 |     set(CMP_GCC 1)
 14 | endif()
 15 | 
 16 | if(WIN32)
 17 |     set(OS_WINDOWS 1)
 18 | else() # If we can't detect an OS, we presume Linux.
 19 |     set(OS_LINUX 1)
 20 | 
 21 |     if(NOT UNIX)
 22 |         message("Attempting to build for an unknown platform. Presuming Linux.")
 23 |     endif()
 24 | endif()
 25 | 
 26 | if(CMP_MSVC)
 27 |     add_definitions(-DCMP_MSVC=1)
 28 | else()
 29 |     add_definitions(-DCMP_MSVC=0)
 30 | endif()
 31 | 
 32 | if(CMP_CLANG)
 33 |     add_definitions(-DCMP_CLANG=1)
 34 | else()
 35 |     add_definitions(-DCMP_CLANG=0)
 36 | endif()
 37 | 
 38 | if(CMP_INTEL)
 39 |     add_definitions(-DCMP_INTEL=1)
 40 | else()
 41 |     add_definitions(-DCMP_INTEL=0)
 42 | endif()
 43 | 
 44 | if(CMP_GCC)
 45 |     add_definitions(-DCMP_GCC=1)
 46 | else()
 47 |     add_definitions(-DCMP_GCC=0)
 48 | endif()
 49 | 
 50 | if(OS_WINDOWS)
 51 |     add_definitions(-DOS_WINDOWS=1)
 52 | else()
 53 |     add_definitions(-DOS_WINDOWS=0)
 54 | endif()
 55 | 
 56 | if(OS_LINUX)
 57 |     add_definitions(-DOS_LINUX=1)
 58 | else()
 59 |     add_definitions(-DOS_LINUX=0)
 60 | endif()
 61 | 
 62 | # Compiler switches
 63 | 
 64 | if(CMP_MSVC)
 65 |     # C++17 support
 66 |     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /std:c++latest")
 67 | 
 68 |     # Multithreaded compilation
 69 |     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /MP")
 70 | 
 71 |     # Disable warnings for usage of compliant functions.
 72 |     add_definitions(-D_CRT_SECURE_NO_WARNINGS)
 73 | 
 74 |     # Release final gets a normal PDB.
 75 |     set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Zi")
 76 |     set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} /DEBUG:FULL")
 77 | 
 78 |     # RelWithDebInfo gets Edit-And-Continue support.
 79 |     set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} /Zi")
 80 |     set(CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} /DEBUG:FASTLINK")
 81 | 
 82 |     # Debug gets Edit-And-Continue support.
 83 |     set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /ZI")
 84 |     set(CMAKE_EXE_LINKER_FLAGS_DEBUG "${CMAKE_EXE_LINKER_FLAGS_DEBUG} /DEBUG:FASTLINK")
 85 | endif()
 86 | 
 87 | if(CMP_GCC OR CMP_CLANG)
 88 |     # C++17 support
 89 |     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -Wall -Wextra -Werror")
 90 | 
 91 |     set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fno-omit-frame-pointer -fsanitize=address")
 92 |     set(CMAKE_LINKER_FLAGS_DEBUG "${CMAKE_LINKER_FLAGS_DEBUG} -fno-omit-frame-pointer -fsanitize=address")
 93 | endif()
 94 | 
 95 | # Define targets
 96 | set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -DTAR_RELEASE_FINAL=1 -DTAR_RELEASE=1 -DTAR_DEBUG=0")
 97 | set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -DTAR_RELEASE_FINAL=0 -DTAR_RELEASE=1 -DTAR_DEBUG=1")
 98 | set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DTAR_RELEASE_FINAL=0 -DTAR_RELEASE=0 -DTAR_DEBUG=1")
 99 | 
100 | set(ARTIFACTS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/Build-Artifacts")
101 | set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${ARTIFACTS_DIR}/bin)
102 | set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${ARTIFACTS_DIR}/lib)
103 | 
104 | set_property(GLOBAL PROPERTY USE_FOLDERS ON)
105 | 
106 | include_directories(${CMAKE_CURRENT_SOURCE_DIR})
107 | 
108 | add_subdirectory(Utility)
109 | add_subdirectory(FileFormats)
110 | add_subdirectory(Examples)
111 | add_subdirectory(Tools)
112 | 


--------------------------------------------------------------------------------
/FileFormats/Tlk/Tlk_Raw.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <cstddef>
 4 | #include <cstdint>
 5 | #include <vector>
 6 | 
 7 | namespace FileFormats::Tlk::Raw {
 8 | 
 9 | // Refer to https://wiki.neverwintervault.org/pages/viewpage.action?pageId=327727
10 | // Specifically, https://wiki.neverwintervault.org/download/attachments/327727/Bioware_Aurora_TalkTable_Format.pdf?api=v2
11 | // Any references to sections in code comments below will refer to this file.
12 | 
13 | // The talk table file, called dialog.tlk (and dialogf.tlk, containing feminine strings for certain
14 | // languages), contains all the strings that the game will display to the user and which therefore need to be
15 | // translated. Keeping all user-visible strings in the talk table makes it easier to produce multiple language
16 | // versions of the game, because all the other game data files (with the exception of voice-over sound
17 | // files) can remain the same between all language versions of the game. Using the talk table also has the
18 | // advantage of reducing the amount of disk space required to store the game, since text for only one
19 | // language is included.
20 | 
21 | struct TlkHeader
22 | {
23 |     char m_FileType[4]; // "TLK "
24 |     char m_FileVersion[4]; // "V3.0"
25 |     std::uint32_t m_LanguageID; // Language ID. See Table 3.2.2
26 |     std::uint32_t m_StringCount; // Number of strings in file
27 |     std::uint32_t m_StringEntriesOffset; // Offset from start of file to the String Entry Table
28 | };
29 | 
30 | struct TlkStringData
31 | {
32 |     // The String Data Table is a list of String Data Elements, each one describing a single string in the
33 |     // dialog.tlk file.
34 |     // The number of elements in the String Data Table is equal to the StringCount specified in the
35 |     // Header of the file. Each element is packed one after another, immediately after the end of the file
36 |     // header.
37 |     // A StringRef is an index into the String Data Table, so StrRef 0 is the first element, StrRef 1 is the
38 |     // second element, and so on.
39 |     // The format of a String Data Element is given in Table 3.3.1
40 | 
41 |     enum StringFlags : std::uint32_t
42 |     {
43 |         // If flag is set, there is text specified in the file for this StrRef.
44 |         // Use the OffsetToString and StringSize to determine what the text is.
45 |         // If flag is unset, then this StrRef has no text. Return an empty string.
46 |         TEXT_PRESENT      = 0x0001,
47 | 
48 |         // If flag is set, read the SoundResRef from the file.
49 |         // If flag is unset, SoundResRef is an empty string
50 |         SND_PRESENT       = 0x0002,
51 | 
52 |         // If flag is set, read the SoundLength from the file.
53 |         // If flag is unset, SoundLength is 0.0 seconds.
54 |         SNDLENGTH_PRESENT = 0x0004
55 |     };
56 | 
57 |     StringFlags m_Flags; // Flags about this StrRef.
58 |     char m_SoundResRef[16]; // ResRef of the wave file associated with this string Unused characters are nulls.
59 |     std::uint32_t m_VolumeVariance; // not used
60 |     std::uint32_t m_PitchVariance; // not used
61 |     std::uint32_t m_OffsetToString; // Offset from StringEntriesOffset to the beginning of the m_StrRef's text.
62 |     std::uint32_t m_StringSize; // Number of bytes in the string.
63 |     float m_SoundLength; // Duration in seconds of the associated wave file
64 | };
65 | 
66 | using TlkStringEntry = std::byte;
67 | 
68 | struct Tlk
69 | {
70 |     TlkHeader m_Header;
71 |     std::vector<TlkStringData> m_StringData;
72 |     std::vector<TlkStringEntry> m_StringEntries;
73 | 
74 |     // Constructs an Tlk from a non-owning pointer.
75 |     static bool ReadFromBytes(std::byte const* bytes, std::size_t bytesCount, Tlk* out);
76 | 
77 |     // Constructs an Tlk from a vector of bytes which we have taken ownership of.
78 |     static bool ReadFromByteVector(std::vector<std::byte>&& bytes, Tlk* out);
79 | 
80 |     // Constructs an Tlk from a file.
81 |     static bool ReadFromFile(char const* path, Tlk* out);
82 | 
83 |     // Writes the raw Tlk to disk.
84 |     bool WriteToFile(char const* path) const;
85 | 
86 | private:
87 |     bool ConstructInternal(std::byte const* bytes, std::size_t bytesCount);
88 |     void ReadStringData(std::byte const* data);
89 |     void ReadStringEntries(std::byte const* data, std::size_t bytesCount);
90 | };
91 | 
92 | }
93 | 


--------------------------------------------------------------------------------
/FileFormats/Bif/Bif_Raw.cpp:
--------------------------------------------------------------------------------
  1 | #include "FileFormats/Bif/Bif_Raw.hpp"
  2 | #include "Utility/Assert.hpp"
  3 | #include "Utility/MemoryMappedFile.hpp"
  4 | #include "Utility/RAIIWrapper.hpp"
  5 | 
  6 | #include <cstring>
  7 | 
  8 | namespace FileFormats::Bif::Raw {
  9 | 
 10 | namespace {
 11 | 
 12 | template <typename T>
 13 | void ReadGenericOffsetable(std::byte const* bytesWithInitialOffset, std::size_t count, std::vector<T>& out)
 14 | {
 15 |     out.resize(count);
 16 |     std::memcpy(out.data(), bytesWithInitialOffset, count * sizeof(T));
 17 | }
 18 | 
 19 | }
 20 | 
 21 | bool Bif::ReadFromBytes(std::byte const* bytes, std::size_t bytesCount, Bif* out)
 22 | {
 23 |     ASSERT(bytes);
 24 |     ASSERT(bytesCount);
 25 |     ASSERT(out);
 26 | 
 27 |     if (!out->ConstructInternal(bytes))
 28 |     {
 29 |         return false;
 30 |     }
 31 | 
 32 |     std::uint32_t offset = out->m_Header.m_VariableTableOffset;
 33 |     offset += out->m_Header.m_VariableResourceCount * sizeof(BifVariableResource);
 34 |     offset += out->m_Header.m_FixedResourceCount * sizeof(BifFixedResource);
 35 | 
 36 |     std::unique_ptr<OwningDataBlock> owningBlock = std::make_unique<OwningDataBlock>();
 37 |     ReadGenericOffsetable(bytes + offset, bytesCount - offset, owningBlock->m_Data);
 38 |     out->m_DataBlock = std::move(owningBlock);
 39 | 
 40 |     return true;
 41 | }
 42 | 
 43 | bool Bif::ReadFromByteVector(std::vector<std::byte>&& bytes, Bif* out)
 44 | {
 45 |     ASSERT(!bytes.empty());
 46 |     ASSERT(out);
 47 | 
 48 |     if (!out->ConstructInternal(bytes.data()))
 49 |     {
 50 |         return false;
 51 |     }
 52 | 
 53 |     std::uint32_t offset = out->m_Header.m_VariableTableOffset;
 54 |     offset += out->m_Header.m_VariableResourceCount * sizeof(BifVariableResource);
 55 |     offset += out->m_Header.m_FixedResourceCount * sizeof(BifFixedResource);
 56 | 
 57 |     std::unique_ptr<NonOwningDataBlock> nonOwningBlock = std::make_unique<NonOwningDataBlock>();
 58 |     nonOwningBlock->m_Data = bytes.data() + offset;
 59 |     nonOwningBlock->m_DataLength = bytes.size() - offset;
 60 |     out->m_DataBlock = std::move(nonOwningBlock);
 61 | 
 62 |     using StorageType = std::vector<std::byte>;
 63 |     out->m_DataBlockStorage = std::make_unique<RAIIWrapper<StorageType>>(std::forward<StorageType>(bytes));
 64 | 
 65 |     return true;
 66 | }
 67 | 
 68 | bool Bif::ReadFromFile(char const* path, Bif* out)
 69 | {
 70 |     ASSERT(path);
 71 |     ASSERT(out);
 72 | 
 73 |     MemoryMappedFile memmap;
 74 |     bool loaded = MemoryMappedFile::MemoryMap(path, &memmap);
 75 | 
 76 |     if (!loaded)
 77 |     {
 78 |         return false;
 79 |     }
 80 | 
 81 |     DataBlock const& memmapped = memmap.GetDataBlock();
 82 | 
 83 |     if (!out->ConstructInternal(memmapped.GetData()))
 84 |     {
 85 |         return false;
 86 |     }
 87 | 
 88 |     std::uint32_t offset = out->m_Header.m_VariableTableOffset;
 89 |     offset += out->m_Header.m_VariableResourceCount * sizeof(BifVariableResource);
 90 |     offset += out->m_Header.m_FixedResourceCount * sizeof(BifFixedResource);
 91 | 
 92 |     std::unique_ptr<NonOwningDataBlock> nonOwningBlock = std::make_unique<NonOwningDataBlock>();
 93 |     nonOwningBlock->m_Data = memmapped.GetData() + offset;
 94 |     nonOwningBlock->m_DataLength = memmapped.GetDataLength() - offset;
 95 |     out->m_DataBlock = std::move(nonOwningBlock);
 96 | 
 97 |     out->m_DataBlockStorage = std::make_unique<RAIIWrapper<MemoryMappedFile>>(std::move(memmap));
 98 | 
 99 |     return true;
100 | }
101 | 
102 | bool Bif::ConstructInternal(std::byte const* bytes)
103 | {
104 |     ASSERT(bytes);
105 | 
106 |     std::memcpy(&m_Header, bytes, sizeof(m_Header));
107 | 
108 |     if (std::memcmp(m_Header.m_FileType, "BIFF", 4) != 0 ||
109 |         std::memcmp(m_Header.m_Version, "V1  ", 4) != 0)
110 |     {
111 |         return false;
112 |     }
113 | 
114 |     ReadVariableResourceTable(bytes);
115 |     ReadFixedResourceTable(bytes);
116 | 
117 |     return true;
118 | }
119 | 
120 | void Bif::ReadVariableResourceTable(std::byte const* data)
121 | {
122 |     std::uint32_t offset = m_Header.m_VariableTableOffset;
123 |     std::uint32_t count = m_Header.m_VariableResourceCount;
124 |     ReadGenericOffsetable(data + offset, count, m_VariableResourceTable);
125 | }
126 | 
127 | void Bif::ReadFixedResourceTable(std::byte const* data)
128 | {
129 |     std::uint32_t offset = m_Header.m_VariableTableOffset;
130 |     offset += m_Header.m_VariableResourceCount * sizeof(BifVariableResource);
131 | 
132 |     std::uint32_t count = m_Header.m_FixedResourceCount;
133 |     ReadGenericOffsetable(data + offset, count, m_FixedResourceTable);
134 | }
135 | 
136 | }
137 | 


--------------------------------------------------------------------------------
/FileFormats/Tlk/Tlk_Friendly.cpp:
--------------------------------------------------------------------------------
  1 | #include "FileFormats/Tlk/Tlk_Friendly.hpp"
  2 | #include "Utility/Assert.hpp"
  3 | 
  4 | #include <cstring>
  5 | #include <string>
  6 | 
  7 | namespace FileFormats::Tlk::Friendly {
  8 | 
  9 | Tlk::Tlk(Raw::Tlk const& rawTlk)
 10 | {
 11 |     m_LanguageId = rawTlk.m_Header.m_LanguageID;
 12 | 
 13 |     for (std::size_t i = 0; i < rawTlk.m_StringData.size(); ++i)
 14 |     {
 15 |         Raw::TlkStringData const& data = rawTlk.m_StringData[i];
 16 | 
 17 |         TlkEntry tlkEntry;
 18 | 
 19 |         if (data.m_Flags & Raw::TlkStringData::TEXT_PRESENT)
 20 |         {
 21 |             ASSERT(data.m_OffsetToString + data.m_StringSize <= rawTlk.m_StringEntries.size());
 22 |             tlkEntry.m_String = std::string(reinterpret_cast<char const*>(rawTlk.m_StringEntries.data() + data.m_OffsetToString), data.m_StringSize);
 23 |         }
 24 | 
 25 |         if (data.m_Flags & Raw::TlkStringData::SND_PRESENT)
 26 |         {
 27 |             tlkEntry.m_SoundResRef = std::string(data.m_SoundResRef, strnlen(data.m_SoundResRef, 16));
 28 |         }
 29 | 
 30 |         if (data.m_Flags & Raw::TlkStringData::SNDLENGTH_PRESENT)
 31 |         {
 32 |             tlkEntry.m_SoundLength = data.m_SoundLength;
 33 |         }
 34 | 
 35 |         m_TlkMap.insert(std::make_pair(static_cast<StrRef>(i), std::move(tlkEntry)));
 36 |     }
 37 | }
 38 | 
 39 | std::string const& Tlk::operator[](StrRef strref) const
 40 | {
 41 |     static const std::string s_EmptyString = "";
 42 |     auto entry = m_TlkMap.find(strref);
 43 | 
 44 |     if (entry == std::end(m_TlkMap) || !entry->second.m_String.has_value())
 45 |     {
 46 |         return s_EmptyString;
 47 |     }
 48 | 
 49 |     return entry->second.m_String.value();
 50 | }
 51 | 
 52 | void Tlk::Set(StrRef strref, TlkEntry value)
 53 | {
 54 |     m_TlkMap[strref] = value;
 55 | }
 56 | 
 57 | std::uint32_t Tlk::GetLanguageId() const
 58 | {
 59 |     return m_LanguageId;
 60 | }
 61 | 
 62 | void Tlk::SetLanguageId(std::uint32_t id)
 63 | {
 64 |     m_LanguageId = id;
 65 | }
 66 | 
 67 | Tlk::TlkMapType::const_iterator Tlk::begin() const
 68 | {
 69 |     return std::cbegin(m_TlkMap);
 70 | }
 71 | 
 72 | Tlk::TlkMapType::const_iterator Tlk::end() const
 73 | {
 74 |     return std::cend(m_TlkMap);
 75 | }
 76 | 
 77 | bool Tlk::WriteToFile(const char* path) const
 78 | {
 79 |     Raw::Tlk rawTlk;
 80 | 
 81 |     std::memcpy(rawTlk.m_Header.m_FileType, "TLK ", 4);
 82 |     std::memcpy(rawTlk.m_Header.m_FileVersion, "V3.0 ", 4);
 83 |     rawTlk.m_Header.m_LanguageID = m_LanguageId;
 84 | 
 85 |     std::uint32_t stringOffset = 0;
 86 | 
 87 |     for (const auto& entry : m_TlkMap)
 88 |     {
 89 |         Raw::TlkStringData stringData;
 90 | 
 91 |         std::uint32_t flags = 0;
 92 | 
 93 |         stringData.m_OffsetToString = 0;
 94 |         stringData.m_StringSize = 0;
 95 | 
 96 |         if (entry.second.m_String.has_value())
 97 |         {
 98 |             flags |= Raw::TlkStringData::StringFlags::TEXT_PRESENT;
 99 | 
100 |             const std::string& str = entry.second.m_String.value();
101 |             stringData.m_OffsetToString = stringOffset;
102 |             stringData.m_StringSize = static_cast<std::uint32_t>(str.size());
103 | 
104 |             rawTlk.m_StringEntries.resize(stringOffset + stringData.m_StringSize);
105 |             std::memcpy(rawTlk.m_StringEntries.data() + stringOffset, str.c_str(), stringData.m_StringSize);
106 | 
107 |             stringOffset += stringData.m_StringSize;
108 |         }
109 | 
110 |         std::memset(stringData.m_SoundResRef, 0, 16);
111 | 
112 |         if (entry.second.m_SoundResRef.has_value())
113 |         {
114 |             flags |= Raw::TlkStringData::StringFlags::SND_PRESENT;
115 |             const char* str = entry.second.m_SoundResRef->c_str();
116 |             std::memcpy(stringData.m_SoundResRef, str, strnlen(str, 16));
117 |         }
118 | 
119 |         stringData.m_SoundLength = 0;
120 | 
121 |         if (entry.second.m_SoundLength.has_value())
122 |         {
123 |             flags |= Raw::TlkStringData::StringFlags::SNDLENGTH_PRESENT;
124 |             stringData.m_SoundLength = entry.second.m_SoundLength.value();
125 |         }
126 | 
127 |         stringData.m_Flags = static_cast<Raw::TlkStringData::StringFlags>(flags);
128 | 
129 |         stringData.m_VolumeVariance = 0;
130 |         stringData.m_PitchVariance = 0;
131 | 
132 |         rawTlk.m_StringData.emplace_back(std::move(stringData));
133 |     }
134 | 
135 |     rawTlk.m_Header.m_StringCount = static_cast<std::uint32_t>(rawTlk.m_StringData.size());
136 |     rawTlk.m_Header.m_StringEntriesOffset = sizeof(rawTlk.m_Header) + (sizeof(rawTlk.m_StringData[0]) * rawTlk.m_Header.m_StringCount);
137 | 
138 |     return rawTlk.WriteToFile(path);
139 | }
140 | 
141 | }
142 | 


--------------------------------------------------------------------------------
/FileFormats/Erf/Erf_Raw.cpp:
--------------------------------------------------------------------------------
  1 | #include "FileFormats/Erf/Erf_Raw.hpp"
  2 | #include "Utility/Assert.hpp"
  3 | #include "Utility/MemoryMappedFile.hpp"
  4 | #include "Utility/RAIIWrapper.hpp"
  5 | 
  6 | #include <cstring>
  7 | 
  8 | namespace {
  9 | 
 10 | template <typename T>
 11 | void ReadGenericOffsetable(std::byte const* bytesWithInitialOffset, std::size_t count, std::vector<T>& out)
 12 | {
 13 |     out.resize(count);
 14 |     std::memcpy(out.data(), bytesWithInitialOffset, count * sizeof(T));
 15 | }
 16 | 
 17 | }
 18 | 
 19 | namespace FileFormats::Erf::Raw {
 20 | 
 21 | bool Erf::ReadFromBytes(std::byte const* bytes, std::size_t bytesCount, Erf* out)
 22 | {
 23 |     ASSERT(bytes);
 24 |     ASSERT(bytesCount);
 25 |     ASSERT(out);
 26 | 
 27 |     if (!out->ConstructInternal(bytes))
 28 |     {
 29 |         return false;
 30 |     }
 31 | 
 32 |     std::uint32_t offset = out->m_Header.m_OffsetToResourceList + (sizeof(ErfResource) * out->m_Header.m_EntryCount); // End of resources block
 33 | 
 34 |     std::unique_ptr<OwningDataBlock> owningBlock = std::make_unique<OwningDataBlock>();
 35 |     ReadGenericOffsetable(bytes + offset, bytesCount - offset, owningBlock->m_Data);
 36 |     out->m_ResourceData = std::move(owningBlock);
 37 | 
 38 |     return true;
 39 | }
 40 | 
 41 | bool Erf::ReadFromByteVector(std::vector<std::byte>&& bytes, Erf* out)
 42 | {
 43 |     ASSERT(!bytes.empty());
 44 |     ASSERT(out);
 45 | 
 46 |     if (!out->ConstructInternal(bytes.data()))
 47 |     {
 48 |         return false;
 49 |     }
 50 | 
 51 |     std::uint32_t offset = out->m_Header.m_OffsetToResourceList + (sizeof(ErfResource) * out->m_Header.m_EntryCount); // End of resources block
 52 | 
 53 |     std::unique_ptr<NonOwningDataBlock> nonOwningBlock = std::make_unique<NonOwningDataBlock>();
 54 |     nonOwningBlock->m_Data = bytes.data() + offset;
 55 |     nonOwningBlock->m_DataLength = bytes.size() - offset;
 56 |     out->m_ResourceData = std::move(nonOwningBlock);
 57 | 
 58 |     using StorageType = std::vector<std::byte>;
 59 |     out->m_DataBlockStorage = std::make_unique<RAIIWrapper<StorageType>>(std::forward<StorageType>(bytes));
 60 | 
 61 |     return true;
 62 | }
 63 | 
 64 | bool Erf::ReadFromFile(char const* path, Erf* out)
 65 | {
 66 |     ASSERT(path);
 67 |     ASSERT(out);
 68 | 
 69 |     MemoryMappedFile memmap;
 70 |     bool loaded = MemoryMappedFile::MemoryMap(path, &memmap);
 71 | 
 72 |     if (!loaded)
 73 |     {
 74 |         return false;
 75 |     }
 76 | 
 77 |     DataBlock const& memmapped = memmap.GetDataBlock();
 78 | 
 79 |     if (!out->ConstructInternal(memmapped.GetData()))
 80 |     {
 81 |         return false;
 82 |     }
 83 | 
 84 |     std::uint32_t offset = out->m_Header.m_OffsetToResourceList + (sizeof(ErfResource) * out->m_Header.m_EntryCount); // End of resources block
 85 | 
 86 |     std::unique_ptr<NonOwningDataBlock> nonOwningBlock = std::make_unique<NonOwningDataBlock>();
 87 |     nonOwningBlock->m_Data = memmapped.GetData() + offset;
 88 |     nonOwningBlock->m_DataLength = memmapped.GetDataLength() - offset;
 89 |     out->m_ResourceData = std::move(nonOwningBlock);
 90 | 
 91 |     out->m_DataBlockStorage = std::make_unique<RAIIWrapper<MemoryMappedFile>>(std::move(memmap));
 92 | 
 93 |     return true;
 94 | }
 95 | 
 96 | 
 97 | bool Erf::ConstructInternal(std::byte const* bytes)
 98 | {
 99 |     std::memcpy(&m_Header, bytes, sizeof(m_Header));
100 | 
101 |     if (std::memcmp(m_Header.m_Version, "V1.0", 4) != 0)
102 |     {
103 |         return false;
104 |     }
105 | 
106 |     ReadLocalisedStrings(bytes);
107 |     ReadKeys(bytes);
108 |     ReadResources(bytes);
109 | 
110 |     ASSERT(m_Keys.size() == m_Resources.size());
111 | 
112 |     return true;
113 | }
114 | 
115 | void Erf::ReadLocalisedStrings(std::byte const* data)
116 | {
117 |     std::byte const* ptr = data + m_Header.m_OffsetToLocalizedString;
118 | 
119 |     for (std::size_t i = 0; i < m_Header.m_LanguageCount; ++i)
120 |     {
121 |         ErfLocalisedString str;
122 |         std::memcpy(&str.m_LanguageId, ptr, sizeof(str.m_LanguageId));
123 |         ptr += sizeof(str.m_LanguageId);
124 | 
125 |         std::uint32_t strSize;
126 |         std::memcpy(&strSize, ptr, sizeof(strSize));
127 |         ptr += sizeof(strSize);
128 | 
129 |         str.m_String = std::string(reinterpret_cast<char const*>(ptr), strSize);
130 | 
131 |         m_LocalisedStrings.emplace_back(str);
132 |     }
133 | }
134 | 
135 | void Erf::ReadKeys(std::byte const* data)
136 | {
137 |     std::uint32_t offset = m_Header.m_OffsetToKeyList;
138 |     std::uint32_t count = m_Header.m_EntryCount;
139 |     ReadGenericOffsetable(data + offset, count, m_Keys);
140 | }
141 | 
142 | void Erf::ReadResources(std::byte const* data)
143 | {
144 |     std::uint32_t offset = m_Header.m_OffsetToResourceList;
145 |     std::uint32_t count = m_Header.m_EntryCount;
146 |     ReadGenericOffsetable(data + offset, count, m_Resources);
147 | }
148 | 
149 | }
150 | 


--------------------------------------------------------------------------------
/FileFormats/Key/Key_Raw.hpp:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include "FileFormats/Resource.hpp"
  4 | 
  5 | #include <cstddef>
  6 | #include <vector>
  7 | 
  8 | namespace FileFormats::Key::Raw {
  9 | 
 10 | // Refer to https://wiki.neverwintervault.org/pages/viewpage.action?pageId=327727
 11 | // Specifically, https://wiki.neverwintervault.org/download/attachments/327727/Bioware_Aurora_KeyBIF_Format.pdf?api=v2
 12 | // Any references to sections in code comments below will refer to this file.
 13 | 
 14 | // A Key file is an index of all the resources contained within a set of BIF files. The key file contains
 15 | // information as to which BIFs it indexes for and what resources are contained in those BIFs.
 16 | 
 17 | struct KeyHeader
 18 | {
 19 |     char m_FileType[4]; // "KEY "
 20 |     char m_FileVersion[4]; // "V1  "
 21 |     std::uint32_t m_BIFCount; // Number of BIF files that this KEY file controls
 22 |     std::uint32_t m_KeyCount; // Number of Resources in all BIF files linked to this keyfile
 23 |     std::uint32_t m_OffsetToFileTable; // Byte offset of File Table from beginning of this file
 24 |     std::uint32_t m_OffsetToKeyTable; // Byte offset of Key Entry Table from beginning of this file
 25 |     std::uint32_t m_BuildYear; // Number of years since 1900
 26 |     std::uint32_t m_BuildDay; // Number of days since January 1
 27 |     std::byte m_Reserved[32]; // 32 bytes Reserved for future use
 28 | };
 29 | 
 30 | struct KeyFile
 31 | {
 32 |     // The File Table is a list of all the BIF files that are associated with the key file.
 33 |     // The number of elements in the File Table is equal to the BIFCount specified in the Header.
 34 |     // Each element in the File Table is a File Entry, and describes a single BIF file.
 35 | 
 36 |     // File size of the BIF.
 37 |     std::uint32_t m_FileSize;
 38 | 
 39 |     // Byte position of the BIF file's filename in this file. Points to a location in the FileName Table.
 40 |     std::uint32_t m_FilenameOffset;
 41 | 
 42 |     // Number of characters in the BIF's filename.
 43 |     std::uint16_t m_FilenameSize;
 44 | 
 45 |     // A number that represents which drives the BIF file is located in. Currently each bit represents a
 46 |     // drive letter. e.g., bit 0 = HD0, which is the directory where the application was installed.
 47 |     std::uint16_t m_Drives;
 48 | };
 49 | 
 50 | // The Filename Table lists the filenames of all the BIF files associated with the key file.
 51 | // Each File Entry in the File Table has a FilenameOffset that indexes into a Filename Entry in the
 52 | // Filename Table.
 53 | 
 54 | // Filename of the BIF as a non-terminated character string.
 55 | // This filename is relative to the the drive where the BIF is located (as specified in the Drives
 56 | // portion of the BIF File Entry).
 57 | // Each Filename must be unique e.g., data\2da.bif
 58 | 
 59 | using KeyFilename = char;
 60 | 
 61 | struct KeyEntry
 62 | {
 63 |     // NOTE: Do not directly memcpy this structure when serialising. It is safe to memcpy between two KeyEntries
 64 |     // but not when you wish to read or write directly into bytes.
 65 |     // There is extra padding which will mess up your sizes and cause issues.
 66 | 
 67 |     // The Key Table is a list of all the resources in all the BIFs associated with this key file.
 68 |     // The number of elements in the Key Table is equal to the KeyCount specified in the Header.
 69 |     // Each element in the Key Table is a Key Entry, and describes a single resource. A resource may be
 70 |     // a Variable Resource, or it may be a Fixed Resource (at this time, all resources are Variable).
 71 | 
 72 |     // The filename of the resource item without its extension.
 73 |     // The game uses this name to access the resource.
 74 |     // Each ResRef must be unique.
 75 |     char m_ResRef[16];
 76 | 
 77 |     // Resource Type of the Resource.
 78 |     Resource::ResourceType m_ResourceType;
 79 | 
 80 |     // A unique ID number. It is generated as follows:
 81 |     // Variable: ID = (x << 20) + y
 82 |     // Fixed: ID = (x << 20) + (y << 14)
 83 |     // x = [Index into File Table to specify a BIF]
 84 |     // y = [Index into Variable or Fixed Resource Table in BIF]
 85 |     // (<< means bit shift left)
 86 |     std::uint32_t m_ResID;
 87 | };
 88 | 
 89 | struct Key
 90 | {
 91 |     KeyHeader m_Header;
 92 |     std::vector<KeyFile> m_Files;
 93 |     std::vector<KeyFilename> m_Filenames;
 94 |     std::vector<KeyEntry> m_Entries;
 95 | 
 96 |     // Constructs an Key from a non-owning pointer.
 97 |     static bool ReadFromBytes(std::byte const* bytes, Key* out);
 98 | 
 99 |     // Constructs an Key from a vector of bytes which we have taken ownership of.
100 |     static bool ReadFromByteVector(std::vector<std::byte>&& bytes, Key* out);
101 | 
102 |     // Constructs an Key from a file.
103 |     static bool ReadFromFile(char const* path, Key* out);
104 | 
105 | private:
106 |     bool ConstructInternal(std::byte const* bytes);
107 |     void ReadFiles(std::byte const* data);
108 |     void ReadFilenames(std::byte const* data);
109 |     void ReadEntries(std::byte const* data);
110 | };
111 | 
112 | }
113 | 


--------------------------------------------------------------------------------
/Tools/Tool_KeyBifExtractor.cpp:
--------------------------------------------------------------------------------
  1 | #include "FileFormats/Bif.hpp"
  2 | #include "FileFormats/Key.hpp"
  3 | #include "Utility/Assert.hpp"
  4 | 
  5 | #include <map>
  6 | 
  7 | #if OS_WINDOWS
  8 |     #include "Windows.h"
  9 | #else
 10 |     #include <sys/stat.h>
 11 | #endif
 12 | 
 13 | namespace {
 14 | 
 15 | // Recursively make the provided directory.
 16 | void RecursivelyEnsureDir(std::string const& dir)
 17 | {
 18 |     for (std::size_t slashIndex = dir.find_first_of("\\/");
 19 |         slashIndex != std::string::npos;
 20 |         slashIndex = dir.find_first_of("\\/", slashIndex + 1))
 21 |     {
 22 |         std::string dirToMake = dir.substr(0, slashIndex);
 23 | 
 24 | #if OS_WINDOWS
 25 |         CreateDirectoryA(dirToMake.c_str(), NULL);
 26 | #else
 27 |         mkdir(dirToMake.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
 28 | #endif
 29 |     }
 30 | }
 31 | 
 32 | int KeyBifExtractor(char* keyPath, char* basePath, char* outPath);
 33 | 
 34 | int KeyBifExtractor(char* keyPath, char* basePath, char* outPath)
 35 | {
 36 |     using namespace FileFormats;
 37 | 
 38 |     // We need to iterate over each BIF in turn, over all resources, which will allow us to extract the data to the correct file name.
 39 |     // The first step is to go over every referenced resource, and sort them into BIF buckets.
 40 |     // We'll extract by BIF. This prevents us from having to open all BIFs up front.
 41 |     std::map<std::size_t, std::vector<Key::Friendly::KeyBifReferencedResource>> resMap;
 42 | 
 43 |     // We also take a copy of the references bifs for use later.
 44 |     std::vector<Key::Friendly::KeyBifReference> bifRefs;
 45 | 
 46 |     {
 47 |         Key::Raw::Key rawKey;
 48 |         bool loaded = Key::Raw::Key::ReadFromFile(keyPath, &rawKey);
 49 | 
 50 |         if (!loaded)
 51 |         {
 52 |             std::printf("Failed to load the KEY file.\n");
 53 |             return 1;
 54 |         }
 55 | 
 56 |         Key::Friendly::Key key(std::move(rawKey));
 57 | 
 58 |         for (Key::Friendly::KeyBifReferencedResource const& res : key.GetReferencedResources())
 59 |         {
 60 |             resMap[res.m_ReferencedBifIndex].emplace_back(res);
 61 |         }
 62 | 
 63 |         bifRefs = key.GetReferencedBifs();
 64 |     }
 65 | 
 66 |     // At this point, we have every resource we care about. Now we iterate over every BIF that the KEY references,
 67 |     // load it up, then extract all the referenced materials to the output path.
 68 |     std::size_t totalExtractedResources = 0;
 69 | 
 70 |     for (std::size_t i = 0; i < bifRefs.size(); ++i)
 71 |     {
 72 |         Key::Friendly::KeyBifReference const& bifref = bifRefs[i];
 73 |         std::string bifPath = std::string(basePath) + "/" + bifref.m_Path;
 74 | 
 75 |         Bif::Raw::Bif rawBif;
 76 |         bool loaded = Bif::Raw::Bif::ReadFromFile(bifPath.c_str(), &rawBif);
 77 |         ASSERT(loaded);
 78 | 
 79 |         if (!loaded)
 80 |         {
 81 |             std::printf("Failed to load the BIF file %s.\n", bifPath.c_str());
 82 |             continue;
 83 |         }
 84 | 
 85 |         // Infer the file name from the path.
 86 |         std::string bifFileName = bifPath;
 87 |         std::size_t lastSlash = bifFileName.find_last_of("\\/");
 88 |         if (lastSlash != std::string::npos)
 89 |         {
 90 |             bifFileName.erase(0, lastSlash + 1);
 91 |         }
 92 | 
 93 |         // Grab the directory for this bif. This will be used later.
 94 |         std::string bifFolder = std::string(outPath) + "/" + bifFileName + "/";
 95 |         RecursivelyEnsureDir(bifFolder);
 96 | 
 97 |         Bif::Friendly::Bif bif(std::move(rawBif));
 98 |         std::size_t extractedResources = 0;
 99 | 
100 |         Bif::Friendly::Bif::BifResourceMap const& bifResMap = bif.GetResources();
101 | 
102 |         // We're iterating over every resource that KEY calls out and that we've assigned to this BIF's bucket.
103 |         for (Key::Friendly::KeyBifReferencedResource const& bifRefRes : resMap[i])
104 |         {
105 |             auto resInBif = bifResMap.find(bifRefRes.m_ReferencedBifResId);
106 |             ASSERT(resInBif != std::end(bifResMap));
107 |             std::string resourcePath = bifFolder + bifRefRes.m_ResRef + "." + Resource::StringFromResourceType(bifRefRes.m_ResType);
108 | 
109 |             FILE* resFile = std::fopen(resourcePath.c_str(), "wb");
110 |             ASSERT(resFile);
111 | 
112 |             if (!resFile)
113 |             {
114 |                 std::printf("Failed to open %s for write.\n", resourcePath.c_str());
115 |                 continue;
116 |             }
117 | 
118 |             std::fwrite(resInBif->second.m_DataBlock->GetData(), resInBif->second.m_DataBlock->GetDataLength(), 1, resFile);
119 |             std::fclose(resFile);
120 | 
121 |             ++extractedResources;
122 |         }
123 | 
124 |         std::printf("Extracted %zu resources from BIF %s to %s\n", extractedResources, bifPath.c_str(), bifFolder.c_str());
125 |         totalExtractedResources += extractedResources;
126 |     }
127 | 
128 |     std::printf("Extracted %zu resources total referenced by KEY %s\n", totalExtractedResources, keyPath);
129 | 
130 |     return 0;
131 | }
132 | 
133 | }
134 | 
135 | // Invoked as such (on Windows)
136 | // "G:/GOG Games/NWN Diamond/chitin.key" "G:/GOG Games/NWN Diamond" "G:/OutPath"
137 | // "G:/BeamdogLibrary/00829/data/nwn_base.key" "G:/BeamdogLibrary/00829" "G:/OutPath"
138 | int main(int argc, char** argv)
139 | {
140 |     if (argc != 4)
141 |     {
142 |         std::printf("key_bif_extractor [keyfilepath] [basegamedir] [outpath]\n");
143 |         return 1;
144 |     }
145 | 
146 |     return KeyBifExtractor(argv[1], argv[2], argv[3]);
147 | }
148 | 


--------------------------------------------------------------------------------
/FileFormats/Bif/Bif_Raw.hpp:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include "FileFormats/Resource.hpp"
  4 | #include "Utility/DataBlock.hpp"
  5 | #include "Utility/VirtualObject.hpp"
  6 | 
  7 | #include <cstddef>
  8 | #include <cstdint>
  9 | #include <memory>
 10 | #include <vector>
 11 | 
 12 | namespace FileFormats::Bif::Raw {
 13 | 
 14 | // Refer to https://wiki.neverwintervault.org/pages/viewpage.action?pageId=327727
 15 | // Specifically, https://wiki.neverwintervault.org/download/attachments/327727/Bioware_Aurora_KeyBIF_Format.pdf?api=v2
 16 | // Any references to sections in code comments below will refer to this file.
 17 | 
 18 | // A BIF contains mutliple resources (files). It does not contain information about each resource's name,
 19 | // and therefore requires its KEY file.
 20 | 
 21 | struct BifHeader
 22 | {
 23 |     char m_FileType[4]; // "BIFF"
 24 |     char m_Version[4]; // "V1"
 25 |     std::uint32_t m_VariableResourceCount; // Number of variable resources in this file.
 26 |     std::uint32_t m_FixedResourceCount; // Number of fixed resources in this file.
 27 |     std::uint32_t m_VariableTableOffset; // Byte position of the Variable Resource Table from beginning of this file.
 28 | };
 29 | 
 30 | struct BifVariableResource
 31 | {
 32 |     // The Variable Resource Table has a number of entries equal to the Variable Resource Count
 33 |     // specified in the Header
 34 | 
 35 |     // A unique ID number. It is generated as follows:
 36 |     // Variable ID = (x << 20) + y
 37 |     // (<< means bit shift left)
 38 |     // y = [Index of this Resource Entry in the BIF]
 39 |     // In the BIFs included with the game CDs, x = y.
 40 |     // In the patch BIFs, x = 0.
 41 |     // This discrepancy in x values does not matter to the game or toolset because
 42 |     // their resource manager system doesn't care about the value of x in a BIF.
 43 |     std::uint32_t m_Id;
 44 | 
 45 |     // The location of the variable resource data. This is a byte
 46 |     // offset from the beginning of the BIF file into the Variable Resource Data block.
 47 |     std::uint32_t m_Offset;
 48 | 
 49 |     // File size of this resource. Specifies the number of bytes
 50 |     // in the Variable Resource Data block that belong to this resource.
 51 |     std::uint32_t m_FileSize;
 52 | 
 53 |     // Resource type of this resource
 54 |     Resource::ResourceType m_ResourceType;
 55 | };
 56 | 
 57 | struct BifFixedResource
 58 | {
 59 |     // NOTE: This block is actually not implemented. Support for Fixed Resources is available, as the
 60 |     // offset is left in the BIF header, but there is currently nothing implemented. As a result, there is no
 61 |     // existing data type for this. Below is what would conceptually become the Fixed resource table.
 62 |     //
 63 |     // The Fixed Resource Table has a number of entries equal to the Fixed Resource Count specified in
 64 |     // the Header. If it has one or more elements, it is located immediately after the end of the Variable
 65 |     // Resource Table. If there are no fixed resources, then this block is not present at all and the
 66 |     // Variable Resource Data block immediately follows the Variable Resource Table.
 67 | 
 68 |     // A unique ID number. It is generated as follows:
 69 |     // Variable ID = (x << 20) + y
 70 |     // (<< means bit shift left)
 71 |     // y = [Index of this Resource Entry in the BIF]
 72 |     // In the BIFs included with the game CDs, x = y.
 73 |     // In the patch BIFs, x = 0.
 74 |     // This discrepancy in x values does not matter to the
 75 |     // game or toolset because their resource manager
 76 |     // system doesn't care about the value of x in a BIF
 77 |     std::uint32_t m_ID;
 78 | 
 79 |     // The location of the fixed resource data. This is a byte
 80 |     // offset from the beginning of the BIF file into the Fixed
 81 |     // Resource Data block.
 82 |     std::uint32_t m_Offset;
 83 | 
 84 |     // Number of parts
 85 |     std::uint32_t m_PartCount;
 86 | 
 87 |     // File size of this resource
 88 |     std::uint32_t m_FileSize;
 89 | 
 90 |     // Resource type of this resource
 91 |     std::uint32_t m_ResourceType;
 92 | };
 93 | 
 94 | using BifDataBlock = DataBlock;
 95 | 
 96 | class Bif
 97 | {
 98 | public:
 99 |     BifHeader m_Header;
100 |     std::vector<BifVariableResource> m_VariableResourceTable;
101 |     std::vector<BifFixedResource> m_FixedResourceTable;
102 | 
103 |     // NOTE: In the spec, this is separated into a variable resource data block and a fixed resource data block.
104 |     // Unfortunately, there's nothing in the header that allows us to observe the size of each of these blocks.
105 |     // Therefore, I am just rolling each block into one big vector for the purposes of this.
106 |     std::unique_ptr<BifDataBlock> m_DataBlock;
107 | 
108 |     // Constructs a Bif from a non-owning pointer. Memory usage may be high.
109 |     static bool ReadFromBytes(std::byte const* bytes, std::size_t bytesCount, Bif* out);
110 | 
111 |     // Constructs a Bif from a vector of bytes which we have taken ownership of. Memory usage will be moderate.
112 |     static bool ReadFromByteVector(std::vector<std::byte>&& bytes, Bif* out);
113 | 
114 |     // Constructs a Bif from a file. The file with be memory mapped so memory usage will be ideal.
115 |     static bool ReadFromFile(char const* path, Bif* out);
116 | 
117 | private:
118 | 
119 |     // This is an RAII wrapper around the various methods of loading a BIF that we have.
120 |     // - If by bytes, this is nullptr.
121 |     // - If by byte vector, this will contain the vector.
122 |     // - If by file, this will contain a handle to the file (since we're memory mapping).
123 |     std::unique_ptr<VirtualObject> m_DataBlockStorage;
124 | 
125 |     bool ConstructInternal(std::byte const* bytes);
126 |     void ReadVariableResourceTable(std::byte const* data);
127 |     void ReadFixedResourceTable(std::byte const* data);
128 | };
129 | 
130 | }
131 | 


--------------------------------------------------------------------------------
/Tools/Tool_GeneratePlaceableBlueprints.cpp:
--------------------------------------------------------------------------------
  1 | #include "FileFormats/2da.hpp"
  2 | #include "FileFormats/Gff.hpp"
  3 | 
  4 | #include <set>
  5 | #include <cstring>
  6 | 
  7 | #if OS_WINDOWS
  8 |     #include "Windows.h"
  9 | #else
 10 |     #include <sys/stat.h>
 11 | #endif
 12 | 
 13 | namespace {
 14 | 
 15 | // Recursively make the provided directory.
 16 | void RecursivelyEnsureDir(std::string const& dir)
 17 | {
 18 |     for (std::size_t slashIndex = dir.find_first_of("\\/");
 19 |         slashIndex != std::string::npos;
 20 |         slashIndex = dir.find_first_of("\\/", slashIndex + 1))
 21 |     {
 22 |         std::string dirToMake = dir.substr(0, slashIndex);
 23 | 
 24 | #if OS_WINDOWS
 25 |         CreateDirectoryA(dirToMake.c_str(), NULL);
 26 | #else
 27 |         mkdir(dirToMake.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
 28 | #endif
 29 |     }
 30 | }
 31 | 
 32 | }
 33 | 
 34 | int GeneratePlaceableBlueprints(const char* twoDAPath, const char* blueprintPath, const char* outputFolderPath, const char* labelFilter)
 35 | {
 36 |     using namespace FileFormats;
 37 | 
 38 |     TwoDA::Raw::TwoDA twoDARaw;
 39 |     if (!TwoDA::Raw::TwoDA::ReadFromFile(twoDAPath, &twoDARaw))
 40 |     {
 41 |         std::printf("Failed to load 2da from %s.\n", twoDAPath);
 42 |         return 1;
 43 |     }
 44 | 
 45 |     Gff::Raw::Gff rawGff;
 46 |     if (!Gff::Raw::Gff::ReadFromFile(blueprintPath, &rawGff))
 47 |     {
 48 |         std::printf("Failed to load the GFF file from %s.\n", blueprintPath);
 49 |         return 1;
 50 |     }
 51 | 
 52 |     TwoDA::Friendly::TwoDA twoDA(std::move(twoDARaw));
 53 |     Gff::Friendly::Gff gff(std::move(rawGff));
 54 | 
 55 |     std::string blueprintPathAsStr = blueprintPath;
 56 |     std::size_t lastDot = blueprintPathAsStr.find_last_of('.');
 57 | 
 58 |     std::string extension;
 59 |     if (lastDot != std::string::npos && lastDot + 1 != blueprintPathAsStr.size())
 60 |     {
 61 |         extension = blueprintPathAsStr.substr(lastDot + 1, blueprintPathAsStr.size() - lastDot - 1);
 62 |     }
 63 |     else
 64 |     {
 65 |         extension = "gff";
 66 |     }
 67 | 
 68 |     // For each row in the 2da, we need to grab:
 69 |     // - RowID -> maps to Appearance in the GFF
 70 |     // - Label -> maps to LocName in the GFF
 71 |     // - ModelName > maps to the filename and also to TemplateResRef in the GFF
 72 | 
 73 |     // Used to enforce based on ModelName that we only have one of the same name.
 74 |     std::set<std::string> uniqueFileNameSet;
 75 | 
 76 |     for (const TwoDA::Friendly::TwoDARow& row : twoDA)
 77 |     {
 78 |         std::uint32_t rowId = row.RowId();
 79 |         std::string label = row.AsStr("Label");
 80 |         std::string modelName = row.AsStr("ModelName");
 81 | 
 82 |         if (labelFilter && label.find(labelFilter) == std::string::npos)
 83 |         {
 84 |             std::printf("%s: Rejecting due to filter mismatch.\n", label.c_str());
 85 |             continue;
 86 |         }
 87 | 
 88 |         Gff::Friendly::Type_DWORD appearance = static_cast<Gff::Friendly::Type_DWORD>(rowId);
 89 | 
 90 |         Gff::Friendly::Type_CExoLocString locName;
 91 |         locName.m_StringRef = 0xFFFFFFFF;
 92 |         locName.m_TotalSize = sizeof(locName.m_StringRef) +
 93 |             sizeof(std::uint32_t); // string count
 94 | 
 95 |         {
 96 |             Gff::Friendly::Type_CExoLocString::SubString substring;
 97 |             substring.m_StringID = 0;
 98 |             substring.m_String = std::move(label);
 99 | 
100 |             locName.m_TotalSize += sizeof(substring.m_StringID) +
101 |                 sizeof(std::uint32_t) + // string size
102 |                 static_cast<std::uint32_t>(substring.m_String.size());
103 | 
104 |             locName.m_SubStrings.emplace_back(std::move(substring));
105 |         }
106 | 
107 |         if (modelName.size() > 14)
108 |         {
109 |             modelName = modelName.substr(0, 14);
110 |         }
111 | 
112 |         if (uniqueFileNameSet.find(modelName) != std::end(uniqueFileNameSet))
113 |         {
114 |             std::string modifiedModelName;
115 |             std::uint8_t i = 0;
116 | 
117 |             do
118 |             {
119 |                 modifiedModelName = modelName + std::to_string(i++);
120 |             } while (uniqueFileNameSet.find(modifiedModelName) != std::end(uniqueFileNameSet));
121 | 
122 |             std::printf("%s: Mapped %s -> %s\n", locName.m_SubStrings[0].m_String.c_str(), modelName.c_str(), modifiedModelName.c_str());
123 | 
124 |             modelName = modifiedModelName;
125 |         }
126 | 
127 |         uniqueFileNameSet.insert(modelName);
128 | 
129 |         Gff::Friendly::Type_CResRef resref;
130 |         resref.m_Size = static_cast<std::uint32_t>(modelName.size());
131 |         std::memcpy(resref.m_String, modelName.data(), resref.m_Size);
132 | 
133 |         gff.GetTopLevelStruct().WriteField("Appearance", appearance);
134 |         gff.GetTopLevelStruct().WriteField("LocName", locName);
135 |         gff.GetTopLevelStruct().WriteField("TemplateResRef", resref);
136 | 
137 |         char pathBuffer[1024];
138 |         std::sprintf(pathBuffer, "%s/%s.%s", outputFolderPath, modelName.c_str(), extension.c_str());
139 |         RecursivelyEnsureDir(pathBuffer);
140 | 
141 |         if (gff.WriteToFile(pathBuffer))
142 |         {
143 |             std::printf("%s: Saved to %s.\n", locName.m_SubStrings[0].m_String.c_str(), pathBuffer);
144 |         }
145 |         else
146 |         {
147 |             std::printf("%s: ERROR: Failed to save to %s.\n", locName.m_SubStrings[0].m_String.c_str(), pathBuffer);
148 |         }
149 |     }
150 | 
151 |     return 0;
152 | }
153 | 
154 | int main(int argc, char** argv)
155 | {
156 |     if (argc < 4)
157 |     {
158 |         std::printf("key_bif_extractor [2da_path] [blueprint_base_path] [output_path] [optional_label_filter]\n");
159 |         return 1;
160 |     }
161 | 
162 |     return GeneratePlaceableBlueprints(argv[1], argv[2], argv[3], argc >= 4 ? argv[4] : nullptr);
163 | }
164 | 


--------------------------------------------------------------------------------
/FileFormats/Resource.hpp:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include <cstdint>
  4 | 
  5 | // This file describes the supported types of resources.
  6 | // The data in this file (and the matching .cpp) is auto generated based on the contents
  7 | // of tables 1.3.1 and 1.3.2 of the Key/BIF spec document. Refer to it for further information.
  8 | // https://wiki.neverwintervault.org/download/attachments/327727/Bioware_Aurora_KeyBIF_Format.pdf?api=v2
  9 | 
 10 | namespace FileFormats::Resource {
 11 | 
 12 | enum class ResourceType : std::uint16_t
 13 | {
 14 |     // Invalid resource type
 15 |     INVALID = 0xFFFF, // N/A
 16 | 
 17 |     // Windows BMP file
 18 |     BMP = 1, // binary
 19 | 
 20 |     // TGA image format
 21 |     TGA = 3, // binary
 22 | 
 23 |     // WAV sound file
 24 |     WAV = 4, // binary
 25 | 
 26 |     // Bioware Packed Layered Texture, used for player character skins, allows for multiple color layers
 27 |     PLT = 6, // binary
 28 | 
 29 |     // Windows INI file format
 30 |     INI = 7, // textini
 31 | 
 32 |     // Text file
 33 |     TXT = 10, // text
 34 | 
 35 |     // Aurora model
 36 |     MDL = 2002, // mdl
 37 | 
 38 |     // NWScript Source
 39 |     NSS = 2009, // text
 40 | 
 41 |     // NWScript Compiled Script
 42 |     NCS = 2010, // binary
 43 | 
 44 |     // BioWare Aurora Engine Area file. Contains information on what tiles are located in an area, as well
 45 |     // as other static area properties that cannot change via scripting. For each .are file in a .mod,
 46 |     // there must also be a corresponding .git and .gic file having the same ResRef.
 47 |     ARE = 2012, // gff
 48 | 
 49 |     // BioWare Aurora Engine Tileset
 50 |     SET = 2013, // textini
 51 | 
 52 |     // Module Info File. See the IFO Format document.
 53 |     IFO = 2014, // gff
 54 | 
 55 |     // Character/Creature
 56 |     BIC = 2015, // gff
 57 | 
 58 |     // Walkmesh
 59 |     WOK = 2016, // mdl
 60 | 
 61 |     // 2-D Array
 62 |     TWODA = 2017, // text
 63 | 
 64 |     // Extra Texture Info
 65 |     TXI = 2022, // text
 66 | 
 67 |     // Game Instance File. Contains information for all object instances in an area, and all area properties
 68 |     // that can change via scripting.
 69 |     GIT = 2023, // gff
 70 | 
 71 |     // Item Blueprint
 72 |     UTI = 2025, // gff
 73 | 
 74 |     // Creature Blueprint
 75 |     UTC = 2027, // gff
 76 | 
 77 |     // Conversation File
 78 |     DLG = 2029, // gff
 79 | 
 80 |     // Tile/Blueprint Palette File
 81 |     ITP = 2030, // gff
 82 | 
 83 |     // Trigger Blueprint
 84 |     UTT = 2032, // gff
 85 | 
 86 |     // Compressed texture file
 87 |     DDS = 2033, // binary
 88 | 
 89 |     // Sound Blueprint
 90 |     UTS = 2035, // gff
 91 | 
 92 |     // Letter-combo probability info for name generation
 93 |     LTR = 2036, // binary
 94 | 
 95 |     // Generic File Format. Used when undesirable to create a new file extension for a resource, but the resource
 96 |     // is a GFF. (Examples of GFFs include itp, utc, uti, ifo, are, git)
 97 |     GFF = 2037, // gff
 98 | 
 99 |     // Faction File
100 |     FAC = 2038, // gff
101 | 
102 |     // Encounter Blueprint
103 |     UTE = 2040, // gff
104 | 
105 |     // Door Blueprint
106 |     UTD = 2042, // gff
107 | 
108 |     // Placeable Object Blueprint
109 |     UTP = 2044, // gff
110 | 
111 |     // Default Values file. Used by area properties dialog
112 |     DFT = 2045, // textini
113 | 
114 |     // Game Instance Comments. Comments on instances are not used by the game, only the toolset, so they are stored
115 |     // in a gic instead of in the git with the other instance properties.
116 |     GIC = 2046, // gff
117 | 
118 |     // Graphical User Interface layout used by game
119 |     GUI = 2047, // gff
120 | 
121 |     // Store/Merchant Blueprint
122 |     UTM = 2051, // gff
123 | 
124 |     // Door walkmesh
125 |     DWK = 2052, // mdl
126 | 
127 |     // Placeable Object walkmesh
128 |     PWK = 2053, // mdl
129 | 
130 |     // Journal File
131 |     JRL = 2056, // gff
132 | 
133 |     // Waypoint Blueprint. See Waypoint GFF document.
134 |     UTW = 2058, // gff
135 | 
136 |     // Sound Set File. See Sound Set File Format document
137 |     SSF = 2060, // binary
138 | 
139 |     // Script Debugger File
140 |     NDB = 2064, // binary
141 | 
142 |     // Plot Manager file/Plot Instance
143 |     PTM = 2065, // gff
144 | 
145 |     // Plot Wizard Blueprint
146 |     PTT = 2066, // gff
147 | 
148 |     // Backup file (for player files)
149 |     BAK = 2067, // gff
150 | 
151 |     // Digital Distribution Dat File
152 |     DAT = 2068, // binary
153 | 
154 |     // Shader files
155 |     SHD = 2069, // binary (I think? Could be text. TODO)
156 | 
157 |     // Backup character files that have already been converted in the oldservervault, rename to BIC to use
158 |     XBC = 2070, // gff
159 | 
160 |     // Webm files
161 |     WBM = 2071, // binary
162 | 
163 |     // Legacy BG Script File
164 |     IDS = 9996, // binary
165 | 
166 |     // Encapsulated file
167 |     ERF = 9997, // binary
168 | 
169 |     // BIFF file
170 |     BIF = 9998, // binary
171 | 
172 |     // Key file
173 |     KEY = 9999 // binary
174 | };
175 | 
176 | enum class ResourceContentType
177 | {
178 |     // Binary file format. Details vary widely as to implementation
179 |     Binary,
180 | 
181 |     // Plain text file. For some text resources, it doesn't matter whether lines are terminated by CR + LF or just
182 |     // CR characters, but for other text resources, it might matter. To avoid complications, always use CR + LF line
183 |     // terminators because that at least will work in all cases.
184 |     Text,
185 | 
186 |     // Windows INI file format. Special case of a text file.
187 |     TextIni,
188 | 
189 |     // BioWare Generic File Format. See the Generic File Format document.
190 |     Gff,
191 | 
192 |     // BioWare Aurora model file format. Can be plain text or binary.
193 |     Mdl
194 | };
195 | 
196 | ResourceContentType ResourceContentTypeFromResourceType(ResourceType res);
197 | ResourceType ResourceTypeFromString(char const* str);
198 | char const* StringFromResourceType(ResourceType res);
199 | 
200 | }
201 | 


--------------------------------------------------------------------------------
/FileFormats/Gff/Gff_Friendly.hpp:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include <any>
  4 | #include <map>
  5 | #include <vector>
  6 | 
  7 | #include "FileFormats/Gff/Gff_Raw.hpp"
  8 | #include "Utility/Assert.hpp"
  9 | 
 10 | namespace FileFormats::Gff::Friendly {
 11 | 
 12 | class GffStruct;
 13 | class GffList;
 14 | 
 15 | using Type_BYTE = Raw::GffField::Type_BYTE;
 16 | using Type_CHAR = Raw::GffField::Type_CHAR;
 17 | using Type_WORD = Raw::GffField::Type_WORD;
 18 | using Type_SHORT = Raw::GffField::Type_SHORT;
 19 | using Type_DWORD = Raw::GffField::Type_DWORD;
 20 | using Type_INT = Raw::GffField::Type_INT;
 21 | using Type_DWORD64 = Raw::GffField::Type_DWORD64;
 22 | using Type_INT64 = Raw::GffField::Type_INT64;
 23 | using Type_FLOAT = Raw::GffField::Type_FLOAT;
 24 | using Type_DOUBLE = Raw::GffField::Type_DOUBLE;
 25 | using Type_CExoString = Raw::GffField::Type_CExoString;
 26 | using Type_CResRef = Raw::GffField::Type_CResRef;
 27 | using Type_CExoLocString = Raw::GffField::Type_CExoLocString;
 28 | using Type_VOID = Raw::GffField::Type_VOID;
 29 | using Type_Struct = GffStruct;
 30 | using Type_List = GffList;
 31 | 
 32 | // Individual functions here rather than a map so users encounter a compile error if they
 33 | // try to write an invalid type when calling from a template.
 34 | Raw::GffField::Type GetTypeFromType(const Type_BYTE&);
 35 | Raw::GffField::Type GetTypeFromType(const Type_CHAR&);
 36 | Raw::GffField::Type GetTypeFromType(const Type_WORD&);
 37 | Raw::GffField::Type GetTypeFromType(const Type_SHORT&);
 38 | Raw::GffField::Type GetTypeFromType(const Type_DWORD&);
 39 | Raw::GffField::Type GetTypeFromType(const Type_INT&);
 40 | Raw::GffField::Type GetTypeFromType(const Type_DWORD64&);
 41 | Raw::GffField::Type GetTypeFromType(const Type_INT64&);
 42 | Raw::GffField::Type GetTypeFromType(const Type_FLOAT&);
 43 | Raw::GffField::Type GetTypeFromType(const Type_DOUBLE&);
 44 | Raw::GffField::Type GetTypeFromType(const Type_CExoString&);
 45 | Raw::GffField::Type GetTypeFromType(const Type_CResRef&);
 46 | Raw::GffField::Type GetTypeFromType(const Type_CExoLocString&);
 47 | Raw::GffField::Type GetTypeFromType(const Type_VOID&);
 48 | Raw::GffField::Type GetTypeFromType(const Type_Struct&);
 49 | Raw::GffField::Type GetTypeFromType(const Type_List&);
 50 | 
 51 | class GffStruct
 52 | {
 53 | public:
 54 |     GffStruct() = default;
 55 | 
 56 |     // This will construct a struct from one struct directly.
 57 |     GffStruct(Raw::GffStruct const& rawStruct, Raw::Gff const& rawGff);
 58 | 
 59 |     // This will construct a struct from one field in the gff - assuming the field is of type struct (e.g. its own entry).
 60 |     GffStruct(Raw::GffField const& rawField, Raw::Gff const& rawGff);
 61 | 
 62 |     // The field map maps between std::string (field name) -> { Type (gff Type), std::any (type safe variant) }
 63 |     using FieldMap = std::map<std::string, std::pair<Raw::GffField::Type, std::any>>;
 64 | 
 65 |     // We expose direct access to the map here. This allows users to iterate over all fields if they need to do so.
 66 |     FieldMap const& GetFields() const;
 67 | 
 68 |     // The mapping of raw types to return values from ReadField matches the defines in Friendly::Type_*.
 69 |     // This looks up the field in m_Fields and extracts the actual type.
 70 |     template <typename T>
 71 |     bool ReadField(std::string const& fieldName, T* out) const;
 72 | 
 73 |     // Similar to above, except using the iterator.
 74 |     template <typename T>
 75 |     static bool ReadField(typename FieldMap::const_iterator iter, T* out);
 76 | 
 77 |     // Similiar to above, except using the kvp pair.
 78 |     template <typename T>
 79 |     static bool ReadField(typename FieldMap::value_type const& kvp, T* out);
 80 | 
 81 |     // The mapping of raw types to GFF types matches the defines in Friendly::Type_*.
 82 |     // This writes into the field the provided value, overwriting it if it already exists.
 83 |     template <typename T>
 84 |     void WriteField(std::string const& fieldName, T field);
 85 | 
 86 |     // Deletes the field if it exists. Does nothing if it does not.
 87 |     // Returns whether the field was deleted.
 88 |     bool DeleteField(std::string const& fieldName);
 89 | 
 90 |     std::uint32_t GetUserDefinedId() const;
 91 |     void SetUserDefinedId(std::uint32_t id);
 92 | 
 93 | private:
 94 |     void ConstructInternal(Raw::GffStruct const& rawStruct, Raw::Gff const& rawGff);
 95 |     void ConstructInternal(std::vector<Raw::GffField> const& rawFields, Raw::Gff const& rawGff);
 96 | 
 97 |     // We map between field name -> variant here.
 98 |     FieldMap m_Fields;
 99 | 
100 |     std::uint32_t m_UserDefinedId;
101 | };
102 | 
103 | template <typename T>
104 | bool GffStruct::ReadField(std::string const& fieldName, T* out) const
105 | {
106 |     ASSERT(out);
107 | 
108 |     auto entry = m_Fields.find(fieldName);
109 |     if (entry != std::end(m_Fields))
110 |     {
111 |         return ReadField(entry, out);
112 |     }
113 | 
114 |     return false;
115 | }
116 | 
117 | template <typename T>
118 | bool GffStruct::ReadField(typename GffStruct::FieldMap::const_iterator iter, T* out)
119 | {
120 |     ASSERT(out);
121 |     return ReadField(*iter, out);
122 | }
123 | 
124 | template <typename T>
125 | bool GffStruct::ReadField(typename GffStruct::FieldMap::value_type const& kvp, T* out)
126 | {
127 |     ASSERT(out);
128 | 
129 |     std::string const& fieldName = kvp.first;
130 |     Raw::GffField::Type type = kvp.second.first;
131 |     std::any const& variant = kvp.second.second;
132 | 
133 |     try
134 |     {
135 |         *out = std::any_cast<T>(variant);
136 |         return true;
137 |     }
138 |     catch (std::bad_cast&)
139 |     {
140 |         ASSERT_FAIL_MSG("Failed to extract field name %s due to a type mismatch. The Gff type stored was %u.", fieldName.c_str(), type);
141 |         return false;
142 |     }
143 | }
144 | 
145 | template <typename T>
146 | void GffStruct::WriteField(std::string const& fieldName, T field)
147 | {
148 |     m_Fields[fieldName] = std::make_pair(GetTypeFromType(field), std::move(field));
149 | }
150 | 
151 | class GffList
152 | {
153 | public:
154 |     GffList() = default;
155 | 
156 |     // Constructs a list from the field describing it.
157 |     GffList(Raw::GffField const& rawField, Raw::Gff const& rawGff);
158 | 
159 |     std::vector<GffStruct>& GetStructs();
160 |     std::vector<GffStruct> const& GetStructs() const;
161 | 
162 | private:
163 |     // Note that these are copies of the structs, rather than references.
164 |     std::vector<GffStruct> m_Structs;
165 | };
166 | 
167 | // This is a user friendly wrapper around the Gff data.
168 | class Gff
169 | {
170 | public:
171 |     Gff();
172 |     Gff(Raw::Gff const& rawGff);
173 | 
174 |     GffStruct& GetTopLevelStruct();
175 |     GffStruct const& GetTopLevelStruct() const;
176 | 
177 |     bool WriteToFile(char const* path) const;
178 | 
179 | private:
180 |     GffStruct m_TopLevelStruct;
181 | };
182 | 
183 | }
184 | 


--------------------------------------------------------------------------------
/FileFormats/Erf/Erf_Raw.hpp:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include "FileFormats/Resource.hpp"
  4 | #include "Utility/DataBlock.hpp"
  5 | #include "Utility/VirtualObject.hpp"
  6 | 
  7 | #include <cstddef>
  8 | #include <cstdint>
  9 | #include <memory>
 10 | #include <string>
 11 | #include <vector>
 12 | 
 13 | namespace FileFormats::Erf::Raw {
 14 | 
 15 | // Refer to https://wiki.neverwintervault.org/pages/viewpage.action?pageId=327727
 16 | // Specifically, https://wiki.neverwintervault.org/download/attachments/327727/Bioware_Aurora_ERF_Format.pdf?api=v2
 17 | // Any references to sections in code comments below will refer to this file.
 18 | //
 19 | // Note that while these structures are the 'raw types', in some cases I have substituted the <size> <contents>
 20 | // pattern with std::vector<> or std::string and I have removed the extra size member. This is to avoid the
 21 | // manual memory management requirements, e.g. so we don't have to allocate and deallocate a char array ourselves.
 22 | 
 23 | struct ErfHeader
 24 | {
 25 |     char m_FileType[4]; // "ERF ", "MOD ", "SAV ", "HAK " as appropriate
 26 |     char m_Version[4]; // "V1.0"
 27 |     std::uint32_t m_LanguageCount; // number of strings in the Localized String Table
 28 |     std::uint32_t m_LocalizedStringSize; // total size (bytes) of Localized String Table
 29 |     std::uint32_t m_EntryCount; // number of files packed into the ERF
 30 |     std::uint32_t m_OffsetToLocalizedString; // from beginning of file, see figure above
 31 |     std::uint32_t m_OffsetToKeyList; // from beginning of file, see figure above
 32 |     std::uint32_t m_OffsetToResourceList; // from beginning of file, see figure above
 33 |     std::uint32_t m_BuildYear; // since 1900
 34 |     std::uint32_t m_BuildDay; // since January 1st
 35 |     std::uint32_t m_DescriptionStrRef; // strref for file description
 36 | 
 37 |     // The Reserved part of the ERF header allows for additional properties to be added to the file format later
 38 |     // while maintaining backward compatibility with older ERFs.
 39 |     std::byte m_Reserved[116];
 40 | };
 41 | 
 42 | struct ErfLocalisedString
 43 | {
 44 |     // The Localized String List is used to provide a description of the ERF. In .mod files, this is where the
 45 |     // module description is stored. For example, during the Load Module screen in NWN (a BioWare Aurora
 46 |     // Engine game), the module descriptions shown in the upper right corner are taken from the Localized String
 47 |     // List. The game obtains the current Language ID from dialog.tlk, and then displays the ERF String whose
 48 |     // LanguageID matches the dialog.tlk language ID.
 49 | 
 50 |     // The String List contains a series of ERF String elements one after another. Note that each element has
 51 |     // a variable size, encoded within the element itself. The LanguageCount from the ERF Header specifies
 52 |     // the number of String List Elements.
 53 | 
 54 |     // Each String List Element has the following structure:
 55 | 
 56 |     std::uint32_t m_LanguageId;
 57 |     //std::uint32_t m_StringSize;
 58 |     //char m_String[m_StringSize];
 59 |     std::string m_String;
 60 | };
 61 | 
 62 | struct ErfKey
 63 | {
 64 |     // The ERF Key List specifies the filename and filetype of all the files packed into the ERF.
 65 | 
 66 |     // The Key List consists of a series of Key structures one after another. Unlike the String List elements,
 67 |     // the Key List elements all have the same size. The EntryCount in the ERF header specifies the number
 68 |     // of Keys.
 69 | 
 70 |     // Each Key List Element has the following structure:
 71 | 
 72 |     char m_ResRef[16]; // Filename
 73 |     std::uint32_t m_ResId; // Resource ID, starts at 0 and increments
 74 |     Resource::ResourceType m_ResType; // File type
 75 |     std::byte m_Reserved[2];
 76 | 
 77 |     // The ResRef is the name of the file with no null terminator and in lower case. A ResRef can only
 78 |     // contain alphanumeric characters or underscores. It must have 1 to 16 characters, and if it contains less
 79 |     // than 16 characters, the remaining ones are nulls.
 80 | 
 81 |     // The ResID in the key structure is redundant, because the its possible to get the ResID for any ERF
 82 |     // Key by subtracting the OffsetToKeyList from its starting address and dividing by the size of a Key
 83 |     // List structure.
 84 | 
 85 |     // When a file is extracted from an ERF, the ResRef is the name of the file after it is extracted, and the
 86 |     // ResType specifies its file extension. For a list of ResTypes, see the section on ResTypes later in this
 87 |     // document.
 88 | 
 89 | };
 90 | 
 91 | struct ErfResource
 92 | {
 93 |     // The Resource List looks just like the Key list, except that it has Resource List elements instead of Key
 94 |     // List elements. The ERF header's EntryCount specifies the number of elements in both the Key List
 95 |     // and the Resource List, and there is a one-to-one correspondence between Keys and Resource List
 96 |     // elements
 97 | 
 98 |     // Each Resource List Element corresponds to a single file packed into the ERF. The Resource structure
 99 |     // specifies where the data for the file begins inside the ERF, and how many bytes of data there are.
100 | 
101 |     std::uint32_t m_OffsetToResource; // offset to file data from beginning of ERF
102 |     std::uint32_t m_ResourceSize; // number of bytes
103 | };
104 | 
105 | using ErfResourceData = DataBlock;
106 | 
107 | struct Erf
108 | {
109 |     ErfHeader m_Header;
110 |     std::vector<ErfLocalisedString> m_LocalisedStrings;
111 |     std::vector<ErfKey> m_Keys;
112 |     std::vector<ErfResource> m_Resources;
113 |     std::unique_ptr<ErfResourceData> m_ResourceData;
114 | 
115 |     // Constructs an Erf from a non-owning pointer. Memory usage may be high.
116 |     static bool ReadFromBytes(std::byte const* bytes, std::size_t bytesCount, Erf* out);
117 | 
118 |     // Constructs an Erf from a vector of bytes which we have taken ownership of. Memory usage will be moderate.
119 |     static bool ReadFromByteVector(std::vector<std::byte>&& bytes, Erf* out);
120 | 
121 |     // Constructs an Erf from a file. The file with be memory mapped so memory usage will be ideal.
122 |     static bool ReadFromFile(char const* path, Erf* out);
123 | 
124 | private:
125 | 
126 |     // This is an RAII wrapper around the various methods of loading a BIF that we have.
127 |     // - If by bytes, this is nullptr.
128 |     // - If by byte vector, this will contain the vector.
129 |     // - If by file, this will contain a handle to the file (since we're memory mapping).
130 |     std::unique_ptr<VirtualObject> m_DataBlockStorage;
131 | 
132 |     bool ConstructInternal(std::byte const* bytes);
133 |     void ReadLocalisedStrings(std::byte const* data);
134 |     void ReadKeys(std::byte const* data);
135 |     void ReadResources(std::byte const* data);
136 |     void ReadResourceData(std::byte const* data, std::size_t bytesCount);
137 | };
138 | 
139 | }


--------------------------------------------------------------------------------
/Tools/Tool_DiffCreature.cpp:
--------------------------------------------------------------------------------
  1 | #include "FileFormats/2da.hpp"
  2 | #include "FileFormats/Gff.hpp"
  3 | 
  4 | #include <format>
  5 | 
  6 | #if OS_WINDOWS
  7 |     #include "Windows.h"
  8 | #else
  9 |     #include <sys/stat.h>
 10 | #endif
 11 | 
 12 | using namespace FileFormats;
 13 | 
 14 | namespace {
 15 | 
 16 | // Recursively make the provided directory.
 17 | void RecursivelyEnsureDir(std::string const& dir)
 18 | {
 19 |     for (std::size_t slashIndex = dir.find_first_of("\\/");
 20 |         slashIndex != std::string::npos;
 21 |         slashIndex = dir.find_first_of("\\/", slashIndex + 1))
 22 |     {
 23 |         std::string dirToMake = dir.substr(0, slashIndex);
 24 | 
 25 | #if OS_WINDOWS
 26 |         CreateDirectoryA(dirToMake.c_str(), NULL);
 27 | #else
 28 |         mkdir(dirToMake.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
 29 | #endif
 30 |     }
 31 | }
 32 | 
 33 | bool g_any_change = false;
 34 | 
 35 | template <typename T>
 36 | std::string DiffField(const char* fieldName, T oldVal, T newVal)
 37 | {
 38 |     if (oldVal != newVal)
 39 |     {
 40 |         g_any_change = true;
 41 |         return std::format("{}: {} -> {}\n", fieldName, oldVal, newVal);
 42 |     }
 43 | 
 44 |     return std::format("{}: {}\n", fieldName, oldVal);
 45 | }
 46 | 
 47 | template <typename T>
 48 | std::string DiffLocalVar(const Gff::Friendly::GffStruct* oldVariable, const Gff::Friendly::GffStruct* newVariable)
 49 | {
 50 |     T oldVal, newVal;
 51 |     newVariable->ReadField("Value", &newVal);
 52 | 
 53 |     if (oldVariable)
 54 |     {
 55 |         oldVariable->ReadField("Value", &oldVal);
 56 |     }
 57 |     else
 58 |     {
 59 |         oldVal = newVal;
 60 |     }
 61 | 
 62 |     return DiffField("Value", oldVal, newVal);
 63 | }
 64 | 
 65 | }
 66 | 
 67 | int DiffCreatures(const char* firstCreaturePath, const char* secondCreaturePath, const char* outputPath)
 68 | {
 69 |     Gff::Raw::Gff firstCreatureGffRaw;
 70 | 
 71 |     if (!Gff::Raw::Gff::ReadFromFile(firstCreaturePath, &firstCreatureGffRaw))
 72 |     {
 73 |         std::printf("Failed to load gff from %s.\n", firstCreaturePath);
 74 |         return 1;
 75 |     }
 76 | 
 77 |     Gff::Raw::Gff secondCreatureGffRaw;
 78 | 
 79 |     if (!Gff::Raw::Gff::ReadFromFile(secondCreaturePath, &secondCreatureGffRaw))
 80 |     {
 81 |         std::printf("Failed to load gff from %s.\n", secondCreaturePath);
 82 |         return 1;
 83 |     }
 84 | 
 85 |     Gff::Friendly::Gff firstCreatureGff(std::move(firstCreatureGffRaw));
 86 |     Gff::Friendly::Gff secondCreatureGff(std::move(secondCreatureGffRaw));
 87 | 
 88 |     std::string output;
 89 | 
 90 |     Gff::Friendly::Type_CExoLocString name;
 91 |     secondCreatureGff.GetTopLevelStruct().ReadField("FirstName", &name);
 92 | 
 93 |     if (!name.m_SubStrings.empty())
 94 |     {
 95 |         output += std::format("{}\n\n", name.m_SubStrings[0].m_String);
 96 |     } 
 97 | 
 98 |     Gff::Friendly::Type_BYTE old_str, new_str;
 99 |     Gff::Friendly::Type_BYTE old_dex, new_dex;
100 |     Gff::Friendly::Type_BYTE old_con, new_con;
101 |     Gff::Friendly::Type_BYTE old_int, new_int;
102 |     Gff::Friendly::Type_BYTE old_wis, new_wis;
103 |     Gff::Friendly::Type_BYTE old_cha, new_cha;
104 |     Gff::Friendly::Type_BYTE old_ac, new_ac;
105 |     Gff::Friendly::Type_SHORT old_max_hp, new_max_hp;
106 |     
107 |     firstCreatureGff.GetTopLevelStruct().ReadField("Str", &old_str);
108 |     secondCreatureGff.GetTopLevelStruct().ReadField("Str", &new_str);
109 | 
110 |     firstCreatureGff.GetTopLevelStruct().ReadField("Dex", &old_dex);
111 |     secondCreatureGff.GetTopLevelStruct().ReadField("Dex", &new_dex);
112 | 
113 |     firstCreatureGff.GetTopLevelStruct().ReadField("Con", &old_con);
114 |     secondCreatureGff.GetTopLevelStruct().ReadField("Con", &new_con);
115 | 
116 |     firstCreatureGff.GetTopLevelStruct().ReadField("Int", &old_int);
117 |     secondCreatureGff.GetTopLevelStruct().ReadField("Int", &new_int);
118 | 
119 |     firstCreatureGff.GetTopLevelStruct().ReadField("Wis", &old_wis);
120 |     secondCreatureGff.GetTopLevelStruct().ReadField("Wis", &new_wis);
121 | 
122 |     firstCreatureGff.GetTopLevelStruct().ReadField("Cha", &old_cha);
123 |     secondCreatureGff.GetTopLevelStruct().ReadField("Cha", &new_cha);
124 | 
125 |     firstCreatureGff.GetTopLevelStruct().ReadField("NaturalAC", &old_ac);
126 |     secondCreatureGff.GetTopLevelStruct().ReadField("NaturalAC", &new_ac);
127 | 
128 |     firstCreatureGff.GetTopLevelStruct().ReadField("MaxHitPoints", &old_max_hp);
129 |     secondCreatureGff.GetTopLevelStruct().ReadField("MaxHitPoints", &new_max_hp);
130 | 
131 |     output += DiffField("Str", old_str, new_str);
132 |     output += DiffField("Dex", old_dex, new_dex);
133 |     output += DiffField("Con", old_con, new_con);
134 |     output += DiffField("Int", old_int, new_int);
135 |     output += DiffField("Wis", old_wis, new_wis);
136 |     output += DiffField("Cha", old_cha, new_cha);
137 |     output += DiffField("NaturalAC", old_ac, new_ac);
138 |     output += DiffField("MaximumHP", old_max_hp, new_max_hp);
139 | 
140 |     output += "\n";
141 | 
142 |     Gff::Friendly::Type_List oldVariables;
143 |     Gff::Friendly::Type_List newVariables;
144 | 
145 |     firstCreatureGff.GetTopLevelStruct().ReadField("VarTable", &oldVariables);
146 |     secondCreatureGff.GetTopLevelStruct().ReadField("VarTable", &newVariables);
147 | 
148 |     for (const Gff::Friendly::GffStruct& variable : newVariables.GetStructs())
149 |     {
150 |         // This code won't cope correctly with two localvars with same name and different type.
151 | 
152 |         Gff::Friendly::Type_CExoString variableName;
153 |         variable.ReadField("Name", &variableName);
154 | 
155 |         Gff::Friendly::Type_DWORD type;
156 |         variable.ReadField("Type", &type);
157 | 
158 |         const Gff::Friendly::GffStruct* oldVariable = nullptr;
159 | 
160 |         // Locate the new variable in the old table to see if it's an add.
161 |         for (const Gff::Friendly::GffStruct& oldVariableCandidate : oldVariables.GetStructs())
162 |         {
163 |             Gff::Friendly::Type_CExoString oldVariableName;
164 |             oldVariableCandidate.ReadField("Name", &oldVariableName);
165 | 
166 |             if (variableName.m_String == oldVariableName.m_String)
167 |             {
168 |                 oldVariable = &oldVariableCandidate;
169 |                 break;
170 |             }
171 |         }
172 | 
173 |         output += std::format("{}LocalVar {} ", oldVariable ? "" : "ADDED ", variableName.m_String);
174 | 
175 |         if (type == 1)
176 |         {
177 |             output += DiffLocalVar<Gff::Friendly::Type_INT>(oldVariable, &variable);
178 |         }
179 |         else if (type == 2)
180 |         {
181 |             output += DiffLocalVar<Gff::Friendly::Type_FLOAT>(oldVariable, &variable);
182 |         }
183 |         else
184 |         {
185 |             output += "UNSUPPORTED TYPE\n";
186 |         }
187 |     }
188 | 
189 |     if (g_any_change)
190 |     {
191 |         RecursivelyEnsureDir(outputPath);
192 |         FILE* f = fopen(outputPath, "w");
193 | 
194 |         if (f)
195 |         {
196 |             fprintf(f, "%s", output.c_str());
197 |             fclose(f);
198 |         }
199 |     }
200 | 
201 |     return 0;
202 | }
203 | 
204 | int main(int argc, char** argv)
205 | {
206 |     if (argc < 4)
207 |     {
208 |         std::printf("diff_creature [first_gff] [second_gff] [output_path]");
209 |         return 1;
210 |     }
211 | 
212 |     return DiffCreatures(argv[1], argv[2], argv[3]);
213 | }
214 | 


--------------------------------------------------------------------------------
/FileFormats/2da/2da_Raw.cpp:
--------------------------------------------------------------------------------
  1 | #include "FileFormats/2da/2da_Raw.hpp"
  2 | #include "Utility/Assert.hpp"
  3 | #include "Utility/MemoryMappedFile.hpp"
  4 | 
  5 | #include <algorithm>
  6 | 
  7 | #include <cstring>
  8 | 
  9 | namespace FileFormats::TwoDA::Raw {
 10 | 
 11 | bool TwoDA::ReadFromBytes(std::byte const* bytes, std::size_t bytesCount, TwoDA* out)
 12 | {
 13 |     ASSERT(bytes);
 14 |     ASSERT(out);
 15 |     return out->ConstructInternal(bytes, bytesCount);
 16 | }
 17 | 
 18 | bool TwoDA::ReadFromByteVector(std::vector<std::byte>&& bytes, TwoDA* out)
 19 | {
 20 |     ASSERT(!bytes.empty());
 21 |     ASSERT(out);
 22 |     return out->ConstructInternal(bytes.data(), bytes.size());
 23 | }
 24 | 
 25 | bool TwoDA::ReadFromFile(char const* path, TwoDA* out)
 26 | {
 27 |     ASSERT(path);
 28 |     ASSERT(out);
 29 | 
 30 |     MemoryMappedFile memmap;
 31 |     bool loaded = MemoryMappedFile::MemoryMap(path, &memmap);
 32 | 
 33 |     if (!loaded)
 34 |     {
 35 |         return false;
 36 |     }
 37 | 
 38 |     return out->ConstructInternal(memmap.GetDataBlock().GetData(), memmap.GetDataBlock().GetDataLength());
 39 | }
 40 | 
 41 | bool TwoDA::WriteToFile(char const* path) const
 42 | {
 43 |     ASSERT(path);
 44 | 
 45 |     FILE* outFile = std::fopen(path, "wb");
 46 | 
 47 |     if (outFile)
 48 |     {
 49 |         // First and second lines - write them out manually.
 50 |         const char* twoDaVersion = "2DA V2.0\n\n";
 51 |         std::fwrite(twoDaVersion, std::strlen(twoDaVersion), 1, outFile);
 52 | 
 53 |         // For each column now, find the greatest width.
 54 |         std::vector<std::size_t> columnWidths;
 55 |         columnWidths.resize(m_Lines[2].m_Tokens.size() + 1);
 56 | 
 57 |         // The column line we handle with a special case, since it's missing row number.
 58 |         for (std::size_t i = 2; i < columnWidths.size(); ++i)
 59 |         {
 60 |             columnWidths[i] = m_Lines[2].m_Tokens[i - 1].size();
 61 |         }
 62 | 
 63 |         for (std::size_t i = 3; i < m_Lines.size(); ++i)
 64 |         {
 65 |             for (std::size_t j = 0; j < columnWidths.size(); ++j)
 66 |             {
 67 |                 const TwoDALine& line = m_Lines[i];
 68 |                 if (j < line.m_Tokens.size())
 69 |                 {
 70 |                     const std::string& token = line.m_Tokens[j];
 71 |                     std::size_t tokenSize = token.size();
 72 |                     if (token.find(" ") != std::string::npos)
 73 |                     {
 74 |                         // Account for quotes
 75 |                         tokenSize += 2;
 76 |                     }
 77 |                     columnWidths[j] = std::max(columnWidths[j], tokenSize);
 78 |                 }
 79 |             }
 80 |         }
 81 | 
 82 |         // Manually print the columns.
 83 |         for (std::size_t i = 0; i < columnWidths.size(); ++i)
 84 |         {
 85 |             const char* str = i == 0 ? "" : m_Lines[2].m_Tokens[i - 1].c_str();
 86 |             std::fprintf(outFile, "%-*s", static_cast<int>(columnWidths[i]), str);
 87 | 
 88 |             if (i != columnWidths.size() - 1)
 89 |             {
 90 |                 std::fwrite(" ", 1, 1, outFile);
 91 |             }
 92 |         }
 93 | 
 94 |         std::fwrite("\n", 1, 1, outFile);
 95 | 
 96 |         // Iterate each row and write it.
 97 |         for (std::size_t i = 3; i < m_Lines.size(); ++i)
 98 |         {
 99 |             for (std::size_t j = 0; j < columnWidths.size(); ++j)
100 |             {
101 |                 std::string str;
102 | 
103 |                 const TwoDALine& line = m_Lines[i];
104 |                 if (j < line.m_Tokens.size())
105 |                 {
106 |                     str = line.m_Tokens[j].c_str();
107 |                     if (str.find(" ") != std::string::npos)
108 |                     {
109 |                         str = "\"" + str + "\"";
110 |                     }
111 |                 }
112 | 
113 |                 std::fprintf(outFile, "%-*s", static_cast<int>(columnWidths[j]), str.c_str());
114 |                 if (j != columnWidths.size() - 1)
115 |                 {
116 |                     std::fwrite(" ", 1, 1, outFile);
117 |                 }
118 |             }
119 | 
120 |             if (i != m_Lines.size())
121 |             {
122 |                 std::fwrite("\n", 1, 1, outFile);
123 |             }
124 |         }
125 | 
126 |         std::fclose(outFile);
127 |         return true;
128 |     }
129 | 
130 |     return false;
131 | }
132 | 
133 | bool TwoDA::ConstructInternal(std::byte const* bytes, std::size_t bytesCount)
134 | {
135 |     // This code is slower than it should be, and allocates.
136 |     // It is possible to do all the parsing in-place but it is difficult to handle the edge cases
137 |     // where the spec has been violated (e.g. wrong line endings, tab characters, pointless trailing white space).
138 | 
139 |     std::vector<char> flattenedLines;
140 |     flattenedLines.resize(bytesCount);
141 |     std::memcpy(flattenedLines.data(), bytes, bytesCount);
142 | 
143 |     std::vector<std::string> lines;
144 | 
145 |     // First pass - split into lines.
146 | 
147 |     for (const char* head = flattenedLines.data(),
148 |         *end = flattenedLines.data() + flattenedLines.size(),
149 |         *tail = head; head < end;)
150 |     {
151 |         while (*head++ != '\n' && head < end) {}
152 |         lines.emplace_back(std::string(tail, head - tail));
153 |         tail = head++;
154 |     }
155 | 
156 |     flattenedLines.clear();
157 | 
158 |     // Second pass -
159 |     // 1. Convert tabs to spaces
160 |     // 2. Remove /r
161 |     // 3. Remove /n
162 | 
163 |     for (std::string& line : lines)
164 |     {
165 |         for (auto iter = std::rbegin(line); iter != std::rend(line); ++iter)
166 |         {
167 |             char ch = *iter;
168 | 
169 |             if (ch == '\n' || ch == '\r')
170 |             {
171 |                 line.erase(std::next(iter).base());
172 |             }
173 |             else if (ch == '\t')
174 |             {
175 |                 *iter = ' ';
176 |             }
177 |         }
178 |     }
179 | 
180 |     // Third pass - now that we have lines set up, we can tokenize them.
181 |     // Anything surrounded by quotes is allowed whitespace, otherwise whitespace is the delimiter.
182 | 
183 |     for (const std::string& line : lines)
184 |     {
185 |         TwoDALine twoDALine;
186 | 
187 |         for (auto head = std::cbegin(line),
188 |             end = std::cend(line), tail = head;
189 |             head != end;)
190 |         {
191 |             bool doingQuotes = false;
192 | 
193 |             while (head != end)
194 |             {
195 |                 char ch = *head++;
196 | 
197 |                 if (ch == '"')
198 |                 {
199 |                     doingQuotes = !doingQuotes;
200 |                 }
201 | 
202 |                 if (!doingQuotes && ch == ' ')
203 |                 {
204 |                     break;
205 |                 }
206 |             }
207 | 
208 |             if (*tail != ' ')
209 |             {
210 |                 auto finalTail = tail;
211 |                 auto finalHead = head;
212 | 
213 |                 if (head != end)
214 |                 {
215 |                     finalHead -= 1;
216 |                 }
217 | 
218 |                 if (*finalTail == '"')
219 |                 {
220 |                     finalTail += 1;
221 |                     finalHead -= 1;
222 |                 }
223 | 
224 |                 twoDALine.m_Tokens.emplace_back(std::string(finalTail, finalHead));
225 |             }
226 | 
227 |             tail = head;
228 |         }
229 | 
230 |         m_Lines.emplace_back(std::move(twoDALine));
231 |     }
232 | 
233 |     ASSERT(m_Lines.size() >= 3);
234 | 
235 |     ASSERT(m_Lines[0].m_Tokens.size() == 2);
236 |     ASSERT(m_Lines[0].m_Tokens[0].size() == 3);
237 |     ASSERT(m_Lines[0].m_Tokens[1].size() == 4);
238 | 
239 |     if (std::memcmp(m_Lines[0].m_Tokens[0].c_str(), "2DA", 3) != 0 ||
240 |         std::memcmp(m_Lines[0].m_Tokens[1].c_str(), "V2.0", 4) != 0)
241 |     {
242 |         return false;
243 |     }
244 | 
245 |     return true;
246 | }
247 | 
248 | }
249 | 


--------------------------------------------------------------------------------
/FileFormats/2da/2da_Raw.hpp:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include <cstddef>
  4 | #include <string>
  5 | #include <vector>
  6 | 
  7 | namespace FileFormats::TwoDA::Raw {
  8 | 
  9 | // Refer to https://wiki.neverwintervault.org/pages/viewpage.action?pageId=327727
 10 | // Specifically, https://wiki.neverwintervault.org/download/attachments/327727/Bioware_Aurora_2DA_Format.pdf?api=v2
 11 | // Any references to sections in code comments below will refer to this file.
 12 | 
 13 | // NOTE: A 2da is a bit different to most other file types. This is not a binary file format - this is a text formats.
 14 | // Therefore, the raw structure will just contain the lines belonging to the file in whitespace-delimited fashion.
 15 | 
 16 | // A 2da file is a plain-text file that describes a 2-dimensional array of data.
 17 | //
 18 | // In BioWare's games, 2da files serve many purposes, and are often crucial to the proper functioning of
 19 | // the game and tools. They describe many aspects of the rules and game engine.
 20 | //
 21 | // Although 2da files are plain text files, they are structured according to a set of rules that must be
 22 | // followed in order for the game and tools to read them correctly.
 23 | 
 24 | // Data types
 25 | // There are three types of data that may be present in a 2da. All data under a given column must be of the
 26 | // same type. The data types are:
 27 | //
 28 | // · String: a string can be any arbitrary sequence of characters. However, if the string contains spaces,
 29 | // then it must be enclosed by quotation mark characters (") because otherwise, the text after the space
 30 | // will be considered to be belong to the next column. The string itself can never contain a quotation
 31 | // mark.
 32 | // · Integer: an integer can be up to 32-bits in size, although the application reading the integer entry is
 33 | // free to assume that the value is actually of a smaller type. For example, boolean values are stored in
 34 | // a 2da as integers, so the column for a boolean property should only contain 0s or 1s.
 35 | // · Float: a 32-bit floating point value.
 36 | //
 37 | // The 2da format does not include data type information for each column because the application that
 38 | // reads the data from the 2da already knows what datatype to assume each column contains.
 39 | //
 40 | // Blank (****) entries
 41 | // The special character sequence **** indicates that an entry contains no data, or the value is not
 42 | // applicable. Note that this character sequence contains exactly 4 asterisk characters, no more and no less.
 43 | // When deleting a row from a 2da file, all columns in that row should be filled with ****s.
 44 | //
 45 | // The **** value is also used to indicate "N/A".
 46 | //
 47 | // An attempt to read a String from a **** entry should return an empty string (""). An attempt to read an
 48 | // Integer or Float should return 0. The programming function that performed the reading operation should
 49 | // indicate that the read attempt failed so that that application knows that the entry value is no ordinary ""
 50 | // or 0.
 51 | 
 52 | using TwoDAToken = std::string;
 53 | 
 54 | struct TwoDALine
 55 | {
 56 |     // Whitespace separating columns
 57 |     // Each column is separated by one or more spaces. The exact number of spaces does not matter, so long
 58 |     // as there is at least one space character. The columns do not have to line up exactly, as shown by row 3
 59 |     // in the example above.
 60 |     //
 61 |     // Important: do not use tab characters to separate columns.
 62 |     //
 63 |     // First column
 64 |     // The first column always contains the row number, with the first row being numbered 0, and all
 65 |     // subsequent rows incrementing upward from there.
 66 |     //
 67 |     // The first column is the only column that does not have a heading.
 68 |     //
 69 |     // Note that the numbering in the first column is for the convenience of the person reading or editing the
 70 |     // 2da file. The game and tools automatically keep track of the index of each row, so if a row is numbered
 71 |     // incorrectly, the game and tools will still use the correct number for the row index. Nevertheless, it is a
 72 |     // good habit to make sure that rows are numbered correctly to avoid confusion.
 73 |     //
 74 |     // Column names
 75 |     // All columns after the first one must have a heading. The heading can be in upper or lower case letters
 76 |     // and may contain underscores.
 77 | 
 78 |     std::vector<TwoDAToken> m_Tokens;
 79 | };
 80 | 
 81 | struct TwoDA
 82 | {
 83 |     // Line 1 - file format version
 84 |     // The first line of a 2da file describes the version of the 2da format followed by the 2da file. The current
 85 |     // version header at the time of this writing is: 2DA V2.0
 86 |     //
 87 |     // Line 2 - blank or optional default
 88 |     // The second line of a 2da file is usually empty.
 89 |     // Optionally, it can specify a default value for all entries in the file. The syntax is: DEFAULT: <text>
 90 |     // where <text> is the default value to use. Note that the default text is subject to the same whitespace
 91 |     // rules as any other column in a 2da. A string containing spaces must therefore be enclosed by quotation
 92 |     // marks.
 93 |     //
 94 |     // The default value will be returned when a requested row and column has no data, such as when asking
 95 |     // for data from a row that does not exist. For String requests, the default text is returned as a string. For
 96 |     // Integer or Floating point requests, the default will be converted to an Integer or Floating point value as
 97 |     // appropriate. If the default string cannot be converted to a numerical type, the return value will be 0. The
 98 |     // programming function that reads the 2da entry should indicate that the read attempt failed.
 99 |     //
100 |     // The default value is not returned when a requested entry is ****. An entry that contains **** will return
101 |     // a blank string or zero.
102 |     //
103 |     // Line 3 - column names
104 |     // The third line of a 2da file contains the names of each column. Each column name is separated from the
105 |     // others by one or more space characters. The exact number of spaces does not matter, so long as there is
106 |     // at least one.
107 |     //
108 |     // A column name contains alphanumeric characters or underscores, and can begin with any of these
109 |     // characters (ie., not restricted to starting with a letter).
110 |     //
111 |     // Lines 4 to infinity - row data
112 |     // All lines after and including line 4 are data rows.
113 |     //
114 |     // Each column in a row is separated from the other columns by one or more space characters. When
115 |     // viewing the contents of a 2da using a fixed-width font, the columns in each row do not have to visually
116 |     // line up with the columns in the other rows, but for ease of reading, it is best to line them up anyway.
117 |     // The very first column in a row is the row's index. The first data row (line 4) has an index of 0, the
118 |     // second data row (line 5) has an index of 1, and so on.
119 |     //
120 |     // Every row must contain the exact same number of columns are there are column names given in line 3,
121 |     // plus one (since the index column has no name).
122 |     //
123 |     // If the data for a column is a string that contains spaces, then the data for that column should begin with
124 |     // a quotation mark and end with a quotation mark. Otherwise, the text after the space will be considered
125 |     // to belong to the next column. Because of how quotation marks are handled, a string entry in a 2da can
126 |     // never contain actually quotation marks itself
127 | 
128 |     std::vector<TwoDALine> m_Lines;
129 | 
130 |     // Constructs a 2da from a non-owning pointer. Memory usage may be high.
131 |     static bool ReadFromBytes(std::byte const* bytes, std::size_t bytesCount, TwoDA* out);
132 | 
133 |     // Constructs a 2da from a vector of bytes which we have taken ownership of. Memory usage will be moderate.
134 |     static bool ReadFromByteVector(std::vector<std::byte>&& bytes, TwoDA* out);
135 | 
136 |     // Constructs a 2da from a file. The file with be memory mapped so memory usage will be ideal.
137 |     static bool ReadFromFile(char const* path, TwoDA* out);
138 | 
139 |     // Writes the raw 2da to disk.
140 |     bool WriteToFile(char const* path) const;
141 | 
142 | private:
143 |     bool ConstructInternal(std::byte const* bytes, std::size_t bytesCount);
144 | };
145 | 
146 | }
147 | 


--------------------------------------------------------------------------------
/FileFormats/2da/2da_Friendly.cpp:
--------------------------------------------------------------------------------
  1 | #include "FileFormats/2da/2da_Friendly.hpp"
  2 | #include "Utility/Assert.hpp"
  3 | 
  4 | namespace FileFormats::TwoDA::Friendly {
  5 | 
  6 | TwoDARow::TwoDARow(std::uint32_t rowId,
  7 |     std::vector<TwoDAEntry>&& data,
  8 |     std::unordered_map<std::string, std::size_t> const& columns)
  9 |     : m_RowId(rowId),
 10 |       m_ColumnNames(columns),
 11 |       m_Data(std::forward<std::vector<TwoDAEntry>>(data))
 12 | {
 13 | }
 14 | 
 15 | TwoDAEntry& TwoDARow::operator[](std::size_t column)
 16 | {
 17 |     ASSERT(column < m_Data.size());
 18 |     return m_Data[column];
 19 | }
 20 | 
 21 | TwoDAEntry& TwoDARow::operator[](std::string const& column)
 22 | {
 23 |     auto columnName = m_ColumnNames.find(column);
 24 |     ASSERT(columnName != std::end(m_ColumnNames));
 25 |     return m_Data[columnName->second];
 26 | }
 27 | 
 28 | TwoDAEntry const& TwoDARow::operator[](std::size_t column) const
 29 | {
 30 |     ASSERT(column < m_Data.size());
 31 |     return m_Data[column];
 32 | }
 33 | 
 34 | TwoDAEntry const& TwoDARow::operator[](std::string const& column) const
 35 | {
 36 |     auto columnName = m_ColumnNames.find(column);
 37 |     ASSERT(columnName != std::end(m_ColumnNames));
 38 |     return m_Data[columnName->second];
 39 | }
 40 | 
 41 | std::string const& TwoDARow::AsStr(std::size_t column) const
 42 | {
 43 |     return operator[](column).m_Data;
 44 | }
 45 | 
 46 | std::string const& TwoDARow::AsStr(std::string const& column) const
 47 | {
 48 |     return operator[](column).m_Data;
 49 | }
 50 | 
 51 | std::int32_t TwoDARow::AsInt(std::size_t column) const
 52 | {
 53 |     return atoi(operator[](column).m_Data.c_str());
 54 | }
 55 | 
 56 | std::int32_t TwoDARow::AsInt(std::string const& column) const
 57 | {
 58 |     return atoi(operator[](column).m_Data.c_str());
 59 | }
 60 | 
 61 | float TwoDARow::AsFloat(std::size_t column) const
 62 | {
 63 |     return static_cast<float>(atof(operator[](column).m_Data.c_str()));
 64 | }
 65 | 
 66 | float TwoDARow::AsFloat(std::string const& column) const
 67 | {
 68 |     return static_cast<float>(atof(operator[](column).m_Data.c_str()));
 69 | }
 70 | 
 71 | bool TwoDARow::IsEmpty(std::size_t column) const
 72 | {
 73 |     return operator[](column).m_IsEmpty;
 74 | }
 75 | 
 76 | bool TwoDARow::IsEmpty(std::string const& column) const
 77 | {
 78 |     return operator[](column).m_IsEmpty;
 79 | }
 80 | 
 81 | std::uint32_t TwoDARow::RowId() const
 82 | {
 83 |     return m_RowId;
 84 | }
 85 | 
 86 | TwoDARow::TwoDAEntries::iterator TwoDARow::begin()
 87 | {
 88 |     return std::begin(m_Data);
 89 | }
 90 | 
 91 | TwoDARow::TwoDAEntries::iterator TwoDARow::end()
 92 | {
 93 |     return std::end(m_Data);
 94 | }
 95 | 
 96 | TwoDARow::TwoDAEntries::const_iterator TwoDARow::begin() const
 97 | {
 98 |     return std::cbegin(m_Data);
 99 | }
100 | 
101 | TwoDARow::TwoDAEntries::const_iterator TwoDARow::end() const
102 | {
103 |     return std::cend(m_Data);
104 | }
105 | 
106 | std::size_t TwoDARow::Size() const
107 | {
108 |     return m_Data.size();
109 | }
110 | 
111 | TwoDA::TwoDA(Raw::TwoDA const& raw2da)
112 | {
113 |     // Line 1 we don't care about ...
114 |     // Line 2 has default values. TODO
115 |     // Line 3 has all the columns.
116 |     ASSERT(raw2da.m_Lines.size() >= 3);
117 | 
118 |     // Iterate over all of the column names and set up the map.
119 |     for (std::size_t i = 0; i < raw2da.m_Lines[2].m_Tokens.size(); ++i)
120 |     {
121 |         std::string const& token = raw2da.m_Lines[2].m_Tokens[i];
122 |         m_ColumnNames[token] = i;
123 |     }
124 | 
125 |     // Iterate over all of the entries and set them up.
126 |     for (std::size_t i = 3; i < raw2da.m_Lines.size(); ++i)
127 |     {
128 |         std::vector<TwoDAEntry> entries;
129 |         std::vector<Raw::TwoDAToken> const& tokens = raw2da.m_Lines[i].m_Tokens;
130 | 
131 |         if (tokens.empty())
132 |         {
133 |             // Non-conforming row - but done a lot in the base game. Just skip it.
134 |             continue;
135 |         }
136 | 
137 |         // We store the row ID - this isn't necessarily to be used by the user,
138 |         // but could store funky stuff that we might want to access.
139 |         std::uint32_t rowId = std::stoul(tokens[0]);
140 | 
141 |         // Skip the first token (which is the row number) when setting this up.
142 |         for (std::size_t j = 1; j < m_ColumnNames.size() + 1; ++j)
143 |         {
144 |             TwoDAEntry entry;
145 | 
146 |             if (j < tokens.size())
147 |             {
148 |                 entry.m_IsEmpty = false;
149 |                 entry.m_Data = tokens[j];
150 |             }
151 |             else
152 |             {
153 |                 entry.m_IsEmpty = true;
154 |             }
155 | 
156 |             entries.emplace_back(std::move(entry));
157 |         }
158 | 
159 |         m_Rows.emplace_back(rowId, std::move(entries), m_ColumnNames);
160 |     }
161 | }
162 | 
163 | std::string const& TwoDA::AsStr(std::size_t row, std::size_t column) const
164 | {
165 |     return m_Rows[row].AsStr(column);
166 | }
167 | 
168 | std::string const& TwoDA::AsStr(std::size_t row, std::string const& column) const
169 | {
170 |     return m_Rows[row].AsStr(column);
171 | }
172 | 
173 | std::int32_t TwoDA::AsInt(std::size_t row, std::size_t column) const
174 | {
175 |     return m_Rows[row].AsInt(column);
176 | }
177 | 
178 | std::int32_t TwoDA::AsInt(std::size_t row, std::string const& column) const
179 | {
180 |     return m_Rows[row].AsInt(column);
181 | }
182 | 
183 | float TwoDA::AsFloat(std::size_t row, std::size_t column) const
184 | {
185 |     return m_Rows[row].AsFloat(column);
186 | }
187 | 
188 | float TwoDA::AsFloat(std::size_t row, std::string const& column) const
189 | {
190 |     return m_Rows[row].AsFloat(column);
191 | }
192 | 
193 | TwoDARow& TwoDA::operator[](std::size_t row)
194 | {
195 |     std::size_t rowSize = m_Rows.size();
196 | 
197 |     if (row < rowSize)
198 |     {
199 |         return m_Rows[row];
200 |     }
201 | 
202 |     std::size_t rowsToAdd = row - rowSize;
203 |     for (std::size_t i = 0; i <= rowsToAdd; ++i)
204 |     {
205 |         std::vector<TwoDAEntry> entries;
206 | 
207 |         for (std::size_t j = 1; j < m_ColumnNames.size() + 1; ++j)
208 |         {
209 |             TwoDAEntry entry;
210 |             entry.m_Data = "****";
211 |             entry.m_IsEmpty = false;
212 |             entries.emplace_back(std::move(entry));
213 |         }
214 | 
215 |         m_Rows.emplace_back(static_cast<std::uint32_t>(rowSize + i), std::move(entries), m_ColumnNames);
216 |     }
217 | 
218 |     return m_Rows[row];
219 | }
220 | 
221 | 
222 | TwoDA::TwoDARows::iterator TwoDA::begin()
223 | {
224 |     return std::begin(m_Rows);
225 | }
226 | 
227 | TwoDA::TwoDARows::iterator TwoDA::end()
228 | {
229 |     return std::end(m_Rows);
230 | }
231 | 
232 | TwoDA::TwoDARows::const_iterator TwoDA::begin() const
233 | {
234 |     return std::cbegin(m_Rows);
235 | }
236 | 
237 | TwoDA::TwoDARows::const_iterator TwoDA::end() const
238 | {
239 |     return std::cend(m_Rows);
240 | }
241 | 
242 | std::size_t TwoDA::Size() const
243 | {
244 |     return m_Rows.size();
245 | }
246 | 
247 | std::unordered_map<std::string, std::size_t> const& TwoDA::GetColumnNames() const
248 | {
249 |     return m_ColumnNames;
250 | }
251 | 
252 | bool TwoDA::WriteToFile(char const* path) const
253 | {
254 |     Raw::TwoDA rawTwoDA;
255 | 
256 |     Raw::TwoDALine firstLine;
257 |     Raw::TwoDALine secondLine;
258 |     firstLine.m_Tokens.emplace_back("2DA V2.0");
259 |     rawTwoDA.m_Lines.emplace_back(std::move(firstLine));
260 |     rawTwoDA.m_Lines.emplace_back(std::move(secondLine));
261 | 
262 |     std::vector<std::string> columnNames;
263 | 
264 |     // Convert column names from map to a flat vector.
265 |     for (auto& kvp : m_ColumnNames)
266 |     {
267 |         if (columnNames.size() < kvp.second + 1)
268 |         {
269 |             columnNames.resize(kvp.second + 1);
270 |         }
271 |         columnNames[kvp.second] = kvp.first;
272 |     }
273 | 
274 |     Raw::TwoDALine columns;
275 | 
276 |     for (const std::string& columnName : columnNames)
277 |     {
278 |         columns.m_Tokens.emplace_back(columnName);
279 |     }
280 | 
281 |     rawTwoDA.m_Lines.emplace_back(std::move(columns));
282 | 
283 |     for (std::size_t rowId = 0; rowId < m_Rows.size(); ++rowId)
284 |     {
285 |         Raw::TwoDALine line;
286 |         line.m_Tokens.emplace_back(std::to_string(rowId));
287 | 
288 |         for (const TwoDAEntry& entry : m_Rows[rowId])
289 |         {
290 |             if (!entry.m_IsEmpty)
291 |             {
292 |                 line.m_Tokens.emplace_back(entry.m_Data);
293 |             }
294 |         }
295 | 
296 |         rawTwoDA.m_Lines.emplace_back(std::move(line));
297 |     }
298 | 
299 |     return rawTwoDA.WriteToFile(path);
300 | }
301 | 
302 | }
303 | 


--------------------------------------------------------------------------------
/Examples/Example_Gff.cpp:
--------------------------------------------------------------------------------
  1 | #include "FileFormats/Gff.hpp"
  2 | #include "Utility/Assert.hpp"
  3 | 
  4 | #include <cstring>
  5 | #include <cinttypes>
  6 | 
  7 | namespace {
  8 | 
  9 | using namespace FileFormats::Gff;
 10 | 
 11 | int GffExample(char* path);
 12 | void GffExamplePrintVarsAndTag(Friendly::Gff const& gff);
 13 | void GffExamplePrintGff_r(Friendly::GffStruct const& element, int depth = 0);
 14 | 
 15 | int GffExample(char* path)
 16 | {
 17 |     // This isn't something that a normal user would want to traverse - this is something that
 18 |     // advanced users may wish to play with, however.
 19 |     Raw::Gff rawGff;
 20 |     bool loaded = Raw::Gff::ReadFromFile(path, &rawGff);
 21 | 
 22 |     std::printf("GFF FileType: %.4s\n", rawGff.m_Header.m_FileType);
 23 |     std::printf("GFF FileVersion: %.4s\n", rawGff.m_Header.m_FileVersion);
 24 | 
 25 |     if (!loaded)
 26 |     {
 27 |         std::printf("Failed to load the GFF file. Check the FileType and FileVersion and ensure the file is well formed.\n");
 28 |         return 1;
 29 |     }
 30 | 
 31 |     // Construct a friendly Gff file from the raw file we loaded earlier.
 32 |     // This provides a much more user friendly interface around the Gff and does not expose
 33 |     // implementation details nor require an advanced understanding of the format to use.
 34 |     Friendly::Gff gff(std::move(rawGff));
 35 | 
 36 |     std::printf("\nPrinting tags and variables.\n");
 37 |     GffExamplePrintVarsAndTag(gff);
 38 | 
 39 |     std::printf("\nPrinting complete Gff data structure.\n");
 40 |     GffExamplePrintGff_r(gff.GetTopLevelStruct());
 41 | 
 42 |     // Then save the GFF back out next to the original with a new field.
 43 |     Friendly::Type_VOID binaryData;
 44 |     binaryData.m_Data.resize(4);
 45 |     std::memcpy(binaryData.m_Data.data(), "BLOB", 4);
 46 |     gff.GetTopLevelStruct().WriteField("MyTestField", binaryData);
 47 | 
 48 |     char pathBuffer[1024];
 49 |     std::sprintf(pathBuffer, "%s.1", path);
 50 |     bool written = gff.WriteToFile(pathBuffer);
 51 |     ASSERT(written);
 52 | 
 53 |     return 0;
 54 | }
 55 | 
 56 | using namespace FileFormats::Gff::Friendly;
 57 | 
 58 | void GffExamplePrintVarsAndTag(Gff const& gff)
 59 | {
 60 |     Type_CExoString tag;
 61 |     if (gff.GetTopLevelStruct().ReadField("Tag", &tag))
 62 |     {
 63 |         std::printf("\nTag: %s\n", tag.m_String.c_str());
 64 |     }
 65 | 
 66 |     Type_List varTable;
 67 |     if (gff.GetTopLevelStruct().ReadField("VarTable", &varTable))
 68 |     {
 69 |         std::vector<Type_Struct> const& entries = varTable.GetStructs();
 70 | 
 71 |         for (std::size_t i = 0; i < entries.size(); ++i)
 72 |         {
 73 |             GffStruct const& entry = entries[i];
 74 |             std::printf("\nVariable #%zu\n", i);
 75 | 
 76 |             Type_CExoString name;
 77 |             entry.ReadField("Name", &name);
 78 |             std::printf(" Name: %s\n", name.m_String.c_str());
 79 | 
 80 |             Type_DWORD type;
 81 |             entry.ReadField("Type", &type);
 82 |             std::printf(" Type: %u\n", type);
 83 | 
 84 |             if (type == 1)
 85 |             {
 86 |                 Type_INT value;
 87 |                 entry.ReadField("Value", &value);
 88 |                 std::printf(" Value: %i\n", value);
 89 |             }
 90 |             else if (type == 2)
 91 |             {
 92 |                 Type_FLOAT value;
 93 |                 entry.ReadField("Value", &value);
 94 |                 std::printf(" Value: %f\n", value);
 95 |             }
 96 |             else if (type == 3)
 97 |             {
 98 |                 Type_CExoString value;
 99 |                 entry.ReadField("Value", &value);
100 |                 std::printf(" Value: %s\n", value.m_String.c_str());
101 |             }
102 |             else if (type == 4)
103 |             {
104 |                 Type_DWORD value;
105 |                 entry.ReadField("Value", &value);
106 |                 std::printf(" Value: %x\n", value);
107 |             }
108 |             else if (type == 5)
109 |             {
110 |                 std::printf(" Value: <location>\n");
111 |             }
112 |             else
113 |             {
114 |                 std::printf(" Value: <unknown>\n");
115 |                 ASSERT_FAIL();
116 |             }
117 |         }
118 |     }
119 | }
120 | 
121 | void GffExamplePrintGff_r(GffStruct const& element, int depth)
122 | {
123 |     if (depth == 0)
124 |     {
125 |         std::printf("\nRoot");
126 |         ++depth;
127 |     }
128 | 
129 |     for (auto const& kvp : element.GetFields())
130 |     {
131 |         // kvp.first = field name
132 |         // kvp.second.first = Raw::GffField::Type
133 |         // kvp.second.second = std::any (we don't care about this one as ReadField will extract for us.
134 | 
135 |         if (kvp.second.first == Raw::GffField::Type::BYTE)
136 |         {
137 |             Type_BYTE value;
138 |             element.ReadField(kvp, &value);
139 |             std::printf("\n%*c%s: [BYTE] %u", depth, ' ', kvp.first.c_str(), value);
140 |         }
141 |         else if (kvp.second.first == Raw::GffField::Type::CHAR)
142 |         {
143 |             Type_CHAR value;
144 |             element.ReadField(kvp, &value);
145 |             std::printf("\n%*c%s: [CHAR] %c", depth, ' ', kvp.first.c_str(), value);
146 |         }
147 |         else if (kvp.second.first == Raw::GffField::Type::WORD)
148 |         {
149 |             Type_WORD value;
150 |             element.ReadField(kvp, &value);
151 |             std::printf("\n%*c%s: [WORD] %u", depth, ' ', kvp.first.c_str(), value);
152 |         }
153 |         else if (kvp.second.first == Raw::GffField::Type::SHORT)
154 |         {
155 |             Type_SHORT value;
156 |             element.ReadField(kvp, &value);
157 |             std::printf("\n%*c%s: [SHORT] %d", depth, ' ', kvp.first.c_str(), value);
158 |         }
159 |         else if (kvp.second.first == Raw::GffField::Type::DWORD)
160 |         {
161 |             Type_DWORD value;
162 |             element.ReadField(kvp, &value);
163 |             std::printf("\n%*c%s: [DWORD] %u", depth, ' ', kvp.first.c_str(), value);
164 |         }
165 |         else if (kvp.second.first == Raw::GffField::Type::INT)
166 |         {
167 |             Type_INT value;
168 |             element.ReadField(kvp, &value);
169 |             std::printf("\n%*c%s: [INT] %d", depth, ' ', kvp.first.c_str(), value);
170 |         }
171 |         else if (kvp.second.first == Raw::GffField::Type::DWORD64)
172 |         {
173 |             Type_DWORD64 value;
174 |             element.ReadField(kvp, &value);
175 |             std::printf("\n%*c%s: [DWORD64] %" PRIu64, depth, ' ', kvp.first.c_str(), value);
176 |         }
177 |         else if (kvp.second.first == Raw::GffField::Type::INT64)
178 |         {
179 |             Type_INT64 value;
180 |             element.ReadField(kvp, &value);
181 |             std::printf("\n%*c%s: [INT64] %" PRId64, depth, ' ', kvp.first.c_str(), value);
182 |         }
183 |         else if (kvp.second.first == Raw::GffField::Type::FLOAT)
184 |         {
185 |             Type_FLOAT value;
186 |             element.ReadField(kvp, &value);
187 |             std::printf("\n%*c%s: [FLOAT] %f", depth, ' ', kvp.first.c_str(), value);
188 |         }
189 |         else if (kvp.second.first == Raw::GffField::Type::DOUBLE)
190 |         {
191 |             Type_DOUBLE value;
192 |             element.ReadField(kvp, &value);
193 |             std::printf("\n%*c%s: [DOUBLE] %f", depth, ' ', kvp.first.c_str(), value);
194 |         }
195 |         else if (kvp.second.first == Raw::GffField::Type::CExoString)
196 |         {
197 |             Type_CExoString value;
198 |             element.ReadField(kvp, &value);
199 |             std::printf("\n%*c%s: [CExoString] %s", depth, ' ', kvp.first.c_str(), value.m_String.c_str());
200 |         }
201 |         else if (kvp.second.first == Raw::GffField::Type::ResRef)
202 |         {
203 |             Type_CResRef value;
204 |             element.ReadField(kvp, &value);
205 |             std::printf("\n%*c%s: [ResRef] %.*s", depth, ' ', kvp.first.c_str(), value.m_Size, value.m_String);
206 |         }
207 |         else if (kvp.second.first == Raw::GffField::Type::CExoLocString)
208 |         {
209 |             Type_CExoLocString value;
210 |             element.ReadField(kvp, &value);
211 |             std::printf("\n%*c%s: [CExoLocString] StrRef: %u, SubString count: %zu", depth, ' ', kvp.first.c_str(), value.m_StringRef, value.m_SubStrings.size());
212 | 
213 |             for (std::size_t i = 0; i < value.m_SubStrings.size(); ++i)
214 |             {
215 |                 Type_CExoLocString::SubString const& substring = value.m_SubStrings[i];
216 |                 std::printf("\n%*c%s #%zu:", depth, ' ', kvp.first.c_str(), i);
217 |                 std::printf("\n%*cStringID: %u", depth + 1, ' ', substring.m_StringID);
218 |                 std::printf("\n%*cString: %s", depth + 1, ' ', substring.m_String.c_str());
219 |             }
220 |         }
221 |         else if (kvp.second.first == Raw::GffField::Type::VOID)
222 |         {
223 |             Type_VOID value;
224 |             element.ReadField(kvp, &value);
225 |             std::printf("\n%*c%s: [VOID] Binary size: %zu", depth, ' ', kvp.first.c_str(), value.m_Data.size());
226 |         }
227 |         else if (kvp.second.first == Raw::GffField::Type::Struct)
228 |         {
229 |             Type_Struct value;
230 |             element.ReadField(kvp, &value);
231 |             std::printf("\n%*c%s: [Struct] Field count: %zu", depth, ' ', kvp.first.c_str(), value.GetFields().size());
232 | 
233 |             GffExamplePrintGff_r(value, depth + 1);
234 |         }
235 |         else if (kvp.second.first == Raw::GffField::Type::List)
236 |         {
237 |             Type_List value;
238 |             element.ReadField(kvp, &value);
239 | 
240 |             std::vector<GffStruct> const& structs = value.GetStructs();
241 |             std::printf("\n%*c%s: [List] Struct count: %zu", depth, ' ', kvp.first.c_str(), structs.size());
242 | 
243 |             for (std::size_t i = 0; i < structs.size(); ++i)
244 |             {
245 |                 std::printf("\n%*c%s #%zu:", depth, ' ', kvp.first.c_str(), i);
246 |                 GffExamplePrintGff_r(structs[i], depth + 1);
247 |             }
248 |         }
249 |         else
250 |         {
251 |             std::printf("\n%*c%s: Unknown gff type: %u", depth, ' ', kvp.first.c_str(), static_cast<std::uint32_t>(kvp.second.first));
252 |             ASSERT_FAIL();
253 |         }
254 |     }
255 | }
256 | 
257 | }
258 | 
259 | int main(int argc, char** argv)
260 | {
261 |     ASSERT(argc == 2);
262 |     return GffExample(argv[1]);
263 | }
264 | 


--------------------------------------------------------------------------------
/FileFormats/Gff/Gff_Raw.cpp:
--------------------------------------------------------------------------------
  1 | #include "FileFormats/Gff/Gff_Raw.hpp"
  2 | #include "Utility/Assert.hpp"
  3 | #include "Utility/MemoryMappedFile.hpp"
  4 | 
  5 | #include <cstring>
  6 | 
  7 | namespace FileFormats::Gff::Raw {
  8 | 
  9 | bool Gff::ReadFromBytes(std::byte const* bytes, Gff* out)
 10 | {
 11 |     ASSERT(bytes);
 12 |     ASSERT(out);
 13 |     return out->ConstructInternal(bytes);
 14 | }
 15 | 
 16 | bool Gff::ReadFromByteVector(std::vector<std::byte>&& bytes, Gff* out)
 17 | {
 18 |     ASSERT(!bytes.empty());
 19 |     ASSERT(out);
 20 |     return out->ConstructInternal(bytes.data());
 21 | }
 22 | 
 23 | bool Gff::ReadFromFile(char const* path, Gff* out)
 24 | {
 25 |     ASSERT(path);
 26 |     ASSERT(out);
 27 | 
 28 |     MemoryMappedFile memmap;
 29 |     bool loaded = MemoryMappedFile::MemoryMap(path, &memmap);
 30 | 
 31 |     if (!loaded)
 32 |     {
 33 |         return false;
 34 |     }
 35 | 
 36 |     return out->ConstructInternal(memmap.GetDataBlock().GetData());
 37 | }
 38 | 
 39 | bool Gff::WriteToFile(char const* path) const
 40 | {
 41 |     ASSERT(path);
 42 | 
 43 |     FILE* outFile = std::fopen(path, "wb");
 44 | 
 45 |     if (outFile)
 46 |     {
 47 |         std::fwrite(&m_Header, sizeof(m_Header), 1, outFile);
 48 |         std::fwrite(m_Structs.data(), sizeof(m_Structs[0]), m_Structs.size(), outFile);
 49 |         std::fwrite(m_Fields.data(), sizeof(m_Fields[0]), m_Fields.size(), outFile);
 50 |         std::fwrite(m_Labels.data(), sizeof(m_Labels[0]), m_Labels.size(), outFile);
 51 |         std::fwrite(m_FieldData.data(), sizeof(m_FieldData[0]), m_FieldData.size(), outFile);
 52 |         std::fwrite(m_FieldIndices.data(), sizeof(m_FieldIndices[0]), m_FieldIndices.size(), outFile);
 53 |         std::fwrite(m_ListIndices.data(), sizeof(m_ListIndices[0]), m_ListIndices.size(), outFile);
 54 |         std::fclose(outFile);
 55 |         return true;
 56 |     }
 57 | 
 58 |     return false;
 59 | }
 60 | 
 61 | namespace {
 62 | 
 63 | template <typename T>
 64 | T ReadSimpleType(GffField const& field, GffField::Type type)
 65 | {
 66 |     ASSERT(field.m_Type == type);
 67 |     T value;
 68 |     std::memcpy(&value, &field.m_DataOrDataOffset, sizeof(value));
 69 |     return value;
 70 | }
 71 | 
 72 | }
 73 | 
 74 | GffField::Type_BYTE Gff::ConstructBYTE(GffField const& field) const
 75 | {
 76 |     return ReadSimpleType<GffField::Type_BYTE>(field, GffField::Type::BYTE);
 77 | }
 78 | 
 79 | GffField::Type_CHAR Gff::ConstructCHAR(GffField const& field) const
 80 | {
 81 |     return ReadSimpleType<GffField::Type_CHAR>(field, GffField::Type::CHAR);
 82 | }
 83 | 
 84 | GffField::Type_WORD Gff::ConstructWORD(GffField const& field) const
 85 | {
 86 |     return ReadSimpleType<GffField::Type_WORD>(field, GffField::Type::WORD);
 87 | }
 88 | 
 89 | GffField::Type_SHORT Gff::ConstructSHORT(GffField const& field) const
 90 | {
 91 |     return ReadSimpleType<GffField::Type_SHORT>(field, GffField::Type::SHORT);
 92 | }
 93 | 
 94 | GffField::Type_DWORD Gff::ConstructDWORD(GffField const& field) const
 95 | {
 96 |     return ReadSimpleType<GffField::Type_DWORD>(field, GffField::Type::DWORD);
 97 | }
 98 | 
 99 | GffField::Type_INT Gff::ConstructINT(GffField const& field) const
100 | {
101 |     return ReadSimpleType<GffField::Type_INT>(field, GffField::Type::INT);
102 | }
103 | 
104 | GffField::Type_DWORD64 Gff::ConstructDWORD64(GffField const& field) const
105 | {
106 |     ASSERT(field.m_Type == GffField::Type::DWORD64);
107 | 
108 |     std::uint32_t offsetIntoFieldDataArray = field.m_DataOrDataOffset;
109 |     ASSERT(offsetIntoFieldDataArray < m_FieldData.size());
110 | 
111 |     GffField::Type_DWORD64 value;
112 |     std::memcpy(&value, m_FieldData.data() + offsetIntoFieldDataArray, sizeof(value));
113 |     return value;
114 | }
115 | 
116 | GffField::Type_INT64 Gff::ConstructINT64(GffField const& field) const
117 | {
118 |     ASSERT(field.m_Type == GffField::Type::INT64);
119 | 
120 |     std::uint32_t offsetIntoFieldDataArray = field.m_DataOrDataOffset;
121 |     ASSERT(offsetIntoFieldDataArray < m_FieldData.size());
122 | 
123 |     GffField::Type_INT64 value;
124 |     std::memcpy(&value, m_FieldData.data() + offsetIntoFieldDataArray, sizeof(value));
125 |     return value;
126 | }
127 | 
128 | GffField::Type_FLOAT Gff::ConstructFLOAT(GffField const& field) const
129 | {
130 |     return ReadSimpleType<GffField::Type_FLOAT>(field, GffField::Type::FLOAT);
131 | }
132 | 
133 | GffField::Type_DOUBLE Gff::ConstructDOUBLE(GffField const& field) const
134 | {
135 |     ASSERT(field.m_Type == GffField::Type::DOUBLE);
136 | 
137 |     std::uint32_t offsetIntoFieldDataArray = field.m_DataOrDataOffset;
138 |     ASSERT(offsetIntoFieldDataArray < m_FieldData.size());
139 | 
140 |     GffField::Type_DOUBLE value;
141 |     std::memcpy(&value, m_FieldData.data() + offsetIntoFieldDataArray, sizeof(value));
142 |     return value;
143 | }
144 | 
145 | GffField::Type_CExoString Gff::ConstructCExoString(GffField const& field) const
146 | {
147 |     ASSERT(field.m_Type == GffField::Type::CExoString);
148 | 
149 |     std::uint32_t offsetIntoFieldDataArray = field.m_DataOrDataOffset;
150 |     ASSERT(offsetIntoFieldDataArray < m_FieldData.size());
151 | 
152 |     GffField::Type_CExoString string;
153 | 
154 |     std::uint32_t length;
155 |     std::memcpy(&length, m_FieldData.data() + offsetIntoFieldDataArray, sizeof(length));
156 | 
157 |     string.m_String = std::string(reinterpret_cast<char const*>(m_FieldData.data() + offsetIntoFieldDataArray + sizeof(length)), length);
158 | 
159 |     return string;
160 | }
161 | 
162 | GffField::Type_CResRef Gff::ConstructResRef(GffField const& field) const
163 | {
164 |     ASSERT(field.m_Type == GffField::Type::ResRef);
165 | 
166 |     std::uint32_t offsetIntoFieldDataArray = field.m_DataOrDataOffset;
167 |     ASSERT(offsetIntoFieldDataArray < m_FieldData.size());
168 | 
169 |     GffField::Type_CResRef resref;
170 | 
171 |     std::memcpy(&resref.m_Size, m_FieldData.data() + offsetIntoFieldDataArray, sizeof(resref.m_Size));
172 |     std::memcpy(&resref.m_String, m_FieldData.data() + offsetIntoFieldDataArray + sizeof(resref.m_Size), sizeof(resref.m_String));
173 | 
174 |     return resref;
175 | }
176 | 
177 | GffField::Type_CExoLocString Gff::ConstructCExoLocString(GffField const& field) const
178 | {
179 |     ASSERT(field.m_Type == GffField::Type::CExoLocString);
180 | 
181 |     std::uint32_t offsetIntoFieldDataArray = field.m_DataOrDataOffset;
182 |     ASSERT(offsetIntoFieldDataArray < m_FieldData.size());
183 | 
184 |     GffField::Type_CExoLocString locString;
185 | 
186 |     std::byte const* ptr = m_FieldData.data() + offsetIntoFieldDataArray;
187 | 
188 |     std::memcpy(&locString.m_TotalSize, ptr, sizeof(locString.m_TotalSize));
189 |     ptr += sizeof(locString.m_TotalSize);
190 | 
191 |     std::memcpy(&locString.m_StringRef, ptr, sizeof(locString.m_StringRef));
192 |     ptr += sizeof(locString.m_StringRef);
193 | 
194 |     std::uint32_t stringCount;
195 |     std::memcpy(&stringCount, ptr, sizeof(stringCount));
196 |     ptr += sizeof(stringCount);
197 | 
198 |     for (std::size_t i = 0; i < stringCount; ++i)
199 |     {
200 |         GffField::Type_CExoLocString::SubString substring;
201 | 
202 |         std::memcpy(&substring.m_StringID, ptr, sizeof(substring.m_StringID));
203 |         ptr += sizeof(substring.m_StringID);
204 | 
205 |         std::uint32_t substringLength;
206 |         std::memcpy(&substringLength, ptr, sizeof(substringLength));
207 |         ptr += sizeof(substringLength);
208 | 
209 |         substring.m_String = std::string(reinterpret_cast<char const*>(ptr), substringLength);
210 |         ptr += substringLength;
211 | 
212 |         locString.m_SubStrings.emplace_back(std::move(substring));
213 |     }
214 | 
215 |     return locString;
216 | }
217 | 
218 | GffField::Type_VOID Gff::ConstructVOID(GffField const& field) const
219 | {
220 |     ASSERT(field.m_Type == GffField::Type::VOID);
221 | 
222 |     std::uint32_t offsetIntoFieldDataArray = field.m_DataOrDataOffset;
223 |     ASSERT(offsetIntoFieldDataArray < m_FieldData.size());
224 | 
225 |     GffField::Type_VOID binary;
226 | 
227 |     std::uint32_t size;
228 |     std::memcpy(&size, m_FieldData.data() + offsetIntoFieldDataArray, sizeof(size));
229 | 
230 |     binary.m_Data.resize(size);
231 |     std::memcpy(binary.m_Data.data(), m_FieldData.data() + offsetIntoFieldDataArray + sizeof(size), size);
232 | 
233 |     return binary;
234 | }
235 | 
236 | GffField::Type_Struct Gff::ConstructStruct(GffField const& field) const
237 | {
238 |     ASSERT(field.m_Type == GffField::Type::Struct);
239 |     return m_Structs[field.m_DataOrDataOffset];
240 | }
241 | 
242 | GffField::Type_List Gff::ConstructList(GffField const& field) const
243 | {
244 |     ASSERT(field.m_Type == GffField::Type::List);
245 | 
246 |     std::uint32_t offsetIntoListIndicesArray = field.m_DataOrDataOffset;
247 |     ASSERT(offsetIntoListIndicesArray < m_ListIndices.size());
248 | 
249 |     GffField::Type_List list;
250 | 
251 |     std::uint32_t length;
252 |     std::memcpy(&length, m_ListIndices.data() + offsetIntoListIndicesArray, sizeof(length));
253 | 
254 |     list.m_Elements.resize(length);
255 |     std::memcpy(list.m_Elements.data(), m_ListIndices.data() + offsetIntoListIndicesArray + sizeof(length), length * sizeof(std::uint32_t));
256 | 
257 |     return list;
258 | }
259 | 
260 | bool Gff::ConstructInternal(std::byte const* bytes)
261 | {
262 |     std::memcpy(&m_Header, bytes, sizeof(m_Header));
263 | 
264 |     if (std::memcmp(m_Header.m_FileVersion, "V3.2", 4) != 0)
265 |     {
266 |         return false;
267 |     }
268 | 
269 |     ReadStructs(bytes);
270 |     ReadFields(bytes);
271 |     ReadLabels(bytes);
272 |     ReadFieldData(bytes);
273 |     ReadFieldIndices(bytes);
274 |     ReadLists(bytes);
275 | 
276 |     return true;
277 | }
278 | 
279 | namespace {
280 | 
281 | template <typename T>
282 | void ReadGenericOffsetable(std::byte const* bytesWithInitialOffset, std::size_t count, std::vector<T>& out)
283 | {
284 |     out.resize(count);
285 |     std::memcpy(out.data(), bytesWithInitialOffset, count * sizeof(T));
286 | }
287 | 
288 | }
289 | 
290 | void Gff::ReadStructs(std::byte const* data)
291 | {
292 |     std::uint32_t offset = m_Header.m_StructOffset;
293 |     std::uint32_t count = m_Header.m_StructCount;
294 |     ReadGenericOffsetable(data + offset, count, m_Structs);
295 | }
296 | 
297 | void Gff::ReadFields(std::byte const* data)
298 | {
299 |     std::uint32_t offset = m_Header.m_FieldOffset;
300 |     std::uint32_t count = m_Header.m_FieldCount;
301 |     ReadGenericOffsetable(data + offset, count, m_Fields);
302 | }
303 | 
304 | void Gff::ReadLabels(std::byte const* data)
305 | {
306 |     std::uint32_t offset = m_Header.m_LabelOffset;
307 |     std::uint32_t count = m_Header.m_LabelCount;
308 |     ReadGenericOffsetable(data + offset, count, m_Labels);
309 | }
310 | 
311 | void Gff::ReadFieldData(std::byte const* data)
312 | {
313 |     std::uint32_t offset = m_Header.m_FieldDataOffset;
314 |     std::uint32_t count = m_Header.m_FieldDataCount;
315 |     ReadGenericOffsetable(data + offset, count, m_FieldData);
316 | }
317 | 
318 | void Gff::ReadFieldIndices(std::byte const* data)
319 | {
320 |     std::uint32_t offset = m_Header.m_FieldIndicesOffset;
321 |     std::uint32_t count = m_Header.m_FieldIndicesCount / sizeof(offset);
322 |     ReadGenericOffsetable(data + offset, count, m_FieldIndices);
323 | }
324 | 
325 | void Gff::ReadLists(std::byte const* data)
326 | {
327 |     std::uint32_t offset = m_Header.m_ListIndicesOffset;
328 |     std::uint32_t count = m_Header.m_ListIndicesCount;
329 |     ReadGenericOffsetable(data + offset, count, m_ListIndices);
330 | }
331 | 
332 | }
333 | 


--------------------------------------------------------------------------------
/FileFormats/Resource.cpp:
--------------------------------------------------------------------------------
  1 | #include "FileFormats/Resource.hpp"
  2 | #include "Utility/Assert.hpp"
  3 | 
  4 | #if OS_WINDOWS
  5 |     #include <cstring>
  6 | #else
  7 |     #include <strings.h>
  8 | #endif
  9 | 
 10 | namespace FileFormats::Resource {
 11 | 
 12 | ResourceContentType ResourceContentTypeFromResourceType(ResourceType res)
 13 | {
 14 |     if (res == ResourceType::BMP) return ResourceContentType::Binary;
 15 |     else if (res == ResourceType::NCS) return ResourceContentType::Binary;
 16 |     else if (res == ResourceType::DDS) return ResourceContentType::Binary;
 17 |     else if (res == ResourceType::LTR) return ResourceContentType::Binary;
 18 |     else if (res == ResourceType::SSF) return ResourceContentType::Binary;
 19 |     else if (res == ResourceType::NDB) return ResourceContentType::Binary;
 20 |     else if (res == ResourceType::TGA) return ResourceContentType::Binary;
 21 |     else if (res == ResourceType::WAV) return ResourceContentType::Binary;
 22 |     else if (res == ResourceType::PLT) return ResourceContentType::Binary;
 23 |     else if (res == ResourceType::DAT) return ResourceContentType::Binary;
 24 |     else if (res == ResourceType::SHD) return ResourceContentType::Binary; // I think? Could be text. TODO
 25 |     else if (res == ResourceType::WBM) return ResourceContentType::Binary;
 26 |     else if (res == ResourceType::IDS) return ResourceContentType::Binary;
 27 |     else if (res == ResourceType::ERF) return ResourceContentType::Binary;
 28 |     else if (res == ResourceType::BIF) return ResourceContentType::Binary;
 29 |     else if (res == ResourceType::KEY) return ResourceContentType::Binary;
 30 |     else if (res == ResourceType::TXT) return ResourceContentType::Text;
 31 |     else if (res == ResourceType::NSS) return ResourceContentType::Text;
 32 |     else if (res == ResourceType::TWODA) return ResourceContentType::Text;
 33 |     else if (res == ResourceType::TXI) return ResourceContentType::Text;
 34 |     else if (res == ResourceType::SET) return ResourceContentType::TextIni;
 35 |     else if (res == ResourceType::DFT) return ResourceContentType::TextIni;
 36 |     else if (res == ResourceType::INI) return ResourceContentType::TextIni;
 37 |     else if (res == ResourceType::ARE) return ResourceContentType::Gff;
 38 |     else if (res == ResourceType::IFO) return ResourceContentType::Gff;
 39 |     else if (res == ResourceType::BIC) return ResourceContentType::Gff;
 40 |     else if (res == ResourceType::GIT) return ResourceContentType::Gff;
 41 |     else if (res == ResourceType::UTI) return ResourceContentType::Gff;
 42 |     else if (res == ResourceType::UTC) return ResourceContentType::Gff;
 43 |     else if (res == ResourceType::DLG) return ResourceContentType::Gff;
 44 |     else if (res == ResourceType::ITP) return ResourceContentType::Gff;
 45 |     else if (res == ResourceType::UTT) return ResourceContentType::Gff;
 46 |     else if (res == ResourceType::UTS) return ResourceContentType::Gff;
 47 |     else if (res == ResourceType::GFF) return ResourceContentType::Gff;
 48 |     else if (res == ResourceType::FAC) return ResourceContentType::Gff;
 49 |     else if (res == ResourceType::UTE) return ResourceContentType::Gff;
 50 |     else if (res == ResourceType::UTD) return ResourceContentType::Gff;
 51 |     else if (res == ResourceType::UTP) return ResourceContentType::Gff;
 52 |     else if (res == ResourceType::GIC) return ResourceContentType::Gff;
 53 |     else if (res == ResourceType::GUI) return ResourceContentType::Gff;
 54 |     else if (res == ResourceType::UTM) return ResourceContentType::Gff;
 55 |     else if (res == ResourceType::JRL) return ResourceContentType::Gff;
 56 |     else if (res == ResourceType::UTW) return ResourceContentType::Gff;
 57 |     else if (res == ResourceType::PTM) return ResourceContentType::Gff;
 58 |     else if (res == ResourceType::PTT) return ResourceContentType::Gff;
 59 |     else if (res == ResourceType::BAK) return ResourceContentType::Gff;
 60 |     else if (res == ResourceType::XBC) return ResourceContentType::Gff;
 61 |     else if (res == ResourceType::MDL) return ResourceContentType::Mdl;
 62 |     else if (res == ResourceType::WOK) return ResourceContentType::Mdl;
 63 |     else if (res == ResourceType::DWK) return ResourceContentType::Mdl;
 64 |     else if (res == ResourceType::PWK) return ResourceContentType::Mdl;
 65 | 
 66 |     ASSERT_FAIL_MSG("Unknown resource type.");
 67 |     return ResourceContentType::Binary;
 68 | }
 69 | 
 70 | ResourceType ResourceTypeFromString(char const* str)
 71 | {
 72 | #if CMP_MSVC && OS_WINDOWS
 73 |     #define CASE_INSENSITIVE_CMP _stricmp
 74 | #else
 75 |     #define CASE_INSENSITIVE_CMP strcasecmp
 76 | #endif
 77 | 
 78 |     if (CASE_INSENSITIVE_CMP(str, "bmp") == 0) return ResourceType::BMP;
 79 |     else if (CASE_INSENSITIVE_CMP(str, "tga") == 0) return ResourceType::TGA;
 80 |     else if (CASE_INSENSITIVE_CMP(str, "wav") == 0) return ResourceType::WAV;
 81 |     else if (CASE_INSENSITIVE_CMP(str, "plt") == 0) return ResourceType::PLT;
 82 |     else if (CASE_INSENSITIVE_CMP(str, "ini") == 0) return ResourceType::INI;
 83 |     else if (CASE_INSENSITIVE_CMP(str, "txt") == 0) return ResourceType::TXT;
 84 |     else if (CASE_INSENSITIVE_CMP(str, "mdl") == 0) return ResourceType::MDL;
 85 |     else if (CASE_INSENSITIVE_CMP(str, "nss") == 0) return ResourceType::NSS;
 86 |     else if (CASE_INSENSITIVE_CMP(str, "ncs") == 0) return ResourceType::NCS;
 87 |     else if (CASE_INSENSITIVE_CMP(str, "are") == 0) return ResourceType::ARE;
 88 |     else if (CASE_INSENSITIVE_CMP(str, "set") == 0) return ResourceType::SET;
 89 |     else if (CASE_INSENSITIVE_CMP(str, "ifo") == 0) return ResourceType::IFO;
 90 |     else if (CASE_INSENSITIVE_CMP(str, "bic") == 0) return ResourceType::BIC;
 91 |     else if (CASE_INSENSITIVE_CMP(str, "wok") == 0) return ResourceType::WOK;
 92 |     else if (CASE_INSENSITIVE_CMP(str, "2da") == 0) return ResourceType::TWODA;
 93 |     else if (CASE_INSENSITIVE_CMP(str, "txi") == 0) return ResourceType::TXI;
 94 |     else if (CASE_INSENSITIVE_CMP(str, "git") == 0) return ResourceType::GIT;
 95 |     else if (CASE_INSENSITIVE_CMP(str, "uti") == 0) return ResourceType::UTI;
 96 |     else if (CASE_INSENSITIVE_CMP(str, "utc") == 0) return ResourceType::UTC;
 97 |     else if (CASE_INSENSITIVE_CMP(str, "dlg") == 0) return ResourceType::DLG;
 98 |     else if (CASE_INSENSITIVE_CMP(str, "itp") == 0) return ResourceType::ITP;
 99 |     else if (CASE_INSENSITIVE_CMP(str, "utt") == 0) return ResourceType::UTT;
100 |     else if (CASE_INSENSITIVE_CMP(str, "dds") == 0) return ResourceType::DDS;
101 |     else if (CASE_INSENSITIVE_CMP(str, "uts") == 0) return ResourceType::UTS;
102 |     else if (CASE_INSENSITIVE_CMP(str, "ltr") == 0) return ResourceType::LTR;
103 |     else if (CASE_INSENSITIVE_CMP(str, "gff") == 0) return ResourceType::GFF;
104 |     else if (CASE_INSENSITIVE_CMP(str, "fac") == 0) return ResourceType::FAC;
105 |     else if (CASE_INSENSITIVE_CMP(str, "ute") == 0) return ResourceType::UTE;
106 |     else if (CASE_INSENSITIVE_CMP(str, "utd") == 0) return ResourceType::UTD;
107 |     else if (CASE_INSENSITIVE_CMP(str, "utp") == 0) return ResourceType::UTP;
108 |     else if (CASE_INSENSITIVE_CMP(str, "dft") == 0) return ResourceType::DFT;
109 |     else if (CASE_INSENSITIVE_CMP(str, "gic") == 0) return ResourceType::GIC;
110 |     else if (CASE_INSENSITIVE_CMP(str, "gui") == 0) return ResourceType::GUI;
111 |     else if (CASE_INSENSITIVE_CMP(str, "utm") == 0) return ResourceType::UTM;
112 |     else if (CASE_INSENSITIVE_CMP(str, "dwk") == 0) return ResourceType::DWK;
113 |     else if (CASE_INSENSITIVE_CMP(str, "pwk") == 0) return ResourceType::PWK;
114 |     else if (CASE_INSENSITIVE_CMP(str, "jrl") == 0) return ResourceType::JRL;
115 |     else if (CASE_INSENSITIVE_CMP(str, "utw") == 0) return ResourceType::UTW;
116 |     else if (CASE_INSENSITIVE_CMP(str, "ssf") == 0) return ResourceType::SSF;
117 |     else if (CASE_INSENSITIVE_CMP(str, "ndb") == 0) return ResourceType::NDB;
118 |     else if (CASE_INSENSITIVE_CMP(str, "ptm") == 0) return ResourceType::PTM;
119 |     else if (CASE_INSENSITIVE_CMP(str, "ptt") == 0) return ResourceType::PTT;
120 |     else if (CASE_INSENSITIVE_CMP(str, "bak") == 0) return ResourceType::BAK;
121 |     else if (CASE_INSENSITIVE_CMP(str, "dat") == 0) return ResourceType::DAT;
122 |     else if (CASE_INSENSITIVE_CMP(str, "shd") == 0) return ResourceType::SHD;
123 |     else if (CASE_INSENSITIVE_CMP(str, "xbc") == 0) return ResourceType::XBC;
124 |     else if (CASE_INSENSITIVE_CMP(str, "wbm") == 0) return ResourceType::WBM;
125 |     else if (CASE_INSENSITIVE_CMP(str, "ids") == 0) return ResourceType::IDS;
126 |     else if (CASE_INSENSITIVE_CMP(str, "erf") == 0) return ResourceType::ERF;
127 |     else if (CASE_INSENSITIVE_CMP(str, "bif") == 0) return ResourceType::BIF;
128 |     else if (CASE_INSENSITIVE_CMP(str, "key") == 0) return ResourceType::KEY;
129 | 
130 |     ASSERT_FAIL_MSG("Unknown resource type.");
131 |     return ResourceType::INVALID;
132 | 
133 | #undef CASE_INSENSITIVE_CMP
134 | }
135 | 
136 | char const* StringFromResourceType(ResourceType res)
137 | {
138 |     if (res == ResourceType::BMP) return "bmp";
139 |     else if (res == ResourceType::TGA) return "tga";
140 |     else if (res == ResourceType::WAV) return "wav";
141 |     else if (res == ResourceType::PLT) return "plt";
142 |     else if (res == ResourceType::INI) return "ini";
143 |     else if (res == ResourceType::TXT) return "txt";
144 |     else if (res == ResourceType::MDL) return "mdl";
145 |     else if (res == ResourceType::NSS) return "nss";
146 |     else if (res == ResourceType::NCS) return "ncs";
147 |     else if (res == ResourceType::ARE) return "are";
148 |     else if (res == ResourceType::SET) return "set";
149 |     else if (res == ResourceType::IFO) return "ifo";
150 |     else if (res == ResourceType::BIC) return "bic";
151 |     else if (res == ResourceType::WOK) return "wok";
152 |     else if (res == ResourceType::TWODA) return "2da";
153 |     else if (res == ResourceType::TXI) return "txi";
154 |     else if (res == ResourceType::GIT) return "git";
155 |     else if (res == ResourceType::UTI) return "uti";
156 |     else if (res == ResourceType::UTC) return "utc";
157 |     else if (res == ResourceType::DLG) return "dlg";
158 |     else if (res == ResourceType::ITP) return "itp";
159 |     else if (res == ResourceType::UTT) return "utt";
160 |     else if (res == ResourceType::DDS) return "dds";
161 |     else if (res == ResourceType::UTS) return "uts";
162 |     else if (res == ResourceType::LTR) return "ltr";
163 |     else if (res == ResourceType::GFF) return "gff";
164 |     else if (res == ResourceType::FAC) return "fac";
165 |     else if (res == ResourceType::UTE) return "ute";
166 |     else if (res == ResourceType::UTD) return "utd";
167 |     else if (res == ResourceType::UTP) return "utp";
168 |     else if (res == ResourceType::DFT) return "dft";
169 |     else if (res == ResourceType::GIC) return "gic";
170 |     else if (res == ResourceType::GUI) return "gui";
171 |     else if (res == ResourceType::UTM) return "utm";
172 |     else if (res == ResourceType::DWK) return "dwk";
173 |     else if (res == ResourceType::PWK) return "pwk";
174 |     else if (res == ResourceType::JRL) return "jrl";
175 |     else if (res == ResourceType::UTW) return "utw";
176 |     else if (res == ResourceType::SSF) return "ssf";
177 |     else if (res == ResourceType::NDB) return "ndb";
178 |     else if (res == ResourceType::PTM) return "ptm";
179 |     else if (res == ResourceType::PTT) return "ptt";
180 |     else if (res == ResourceType::BAK) return "bak";
181 |     else if (res == ResourceType::DAT) return "dat";
182 |     else if (res == ResourceType::SHD) return "shd";
183 |     else if (res == ResourceType::XBC) return "xbc";
184 |     else if (res == ResourceType::WBM) return "wbm";
185 |     else if (res == ResourceType::IDS) return "ids";
186 |     else if (res == ResourceType::ERF) return "erf";
187 |     else if (res == ResourceType::BIF) return "bif";
188 |     else if (res == ResourceType::KEY) return "key";
189 | 
190 |     ASSERT_FAIL_MSG("Unknown resource type.");
191 |     return "unk";
192 | }
193 | 
194 | }


--------------------------------------------------------------------------------
/FileFormats/Gff/Gff_Raw.hpp:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include <cstddef>
  4 | #include <cstdint>
  5 | #include <string>
  6 | #include <vector>
  7 | 
  8 | namespace FileFormats::Gff::Raw {
  9 | 
 10 | // Refer to https://wiki.neverwintervault.org/pages/viewpage.action?pageId=327727
 11 | // Specifically, https://wiki.neverwintervault.org/download/attachments/327727/Bioware_Aurora_GFF_Format.pdf?api=v2
 12 | // Any references to sections in code comments below will refer to this file.
 13 | //
 14 | // Note that while these structures are the 'raw types', in some cases I have substituted the <size> <contents>
 15 | // pattern with std::vector<> or std::string and I have removed the extra size member. This is to avoid the
 16 | // manual memory management requirements, e.g. so we don't have to allocate and deallocate a char array ourselves.
 17 | 
 18 | struct GffHeader
 19 | {
 20 |     // The GFF header contains a number of values, all of them DWORDs (32-bit unsigned integers). The
 21 |     // header contains offset information for all the other sections in the GFF file.
 22 |     // Values in the header are as follows, and arranged in the order listed :
 23 | 
 24 |     char m_FileType[4]; // 4-char file type string
 25 |     char m_FileVersion[4]; // 4-char GFF Version. At the time of writing, the version is "V3.2"
 26 | 
 27 |     std::uint32_t m_StructOffset; // Offset of Struct array as bytes from the beginning of the file
 28 |     std::uint32_t m_StructCount; // Number of elements in Struct array
 29 |     std::uint32_t m_FieldOffset; // Offset of Field array as bytes from the beginning of the file
 30 |     std::uint32_t m_FieldCount; // Number of elements in Field array
 31 |     std::uint32_t m_LabelOffset; // Offset of Label array as bytes from the beginning of the file
 32 |     std::uint32_t m_LabelCount; // Number of elements in Label array
 33 |     std::uint32_t m_FieldDataOffset; // Offset of Field Data as bytes from the beginning of the file
 34 |     std::uint32_t m_FieldDataCount; // Number of bytes in Field Data block
 35 |     std::uint32_t m_FieldIndicesOffset; // Offset of Field Indices array as bytes from the beginning of the file
 36 |     std::uint32_t m_FieldIndicesCount; // Number of bytes in Field Indices array
 37 |     std::uint32_t m_ListIndicesOffset; // Offset of List Indices array as bytes from the beginning of the file
 38 |     std::uint32_t m_ListIndicesCount; // Number of bytes in List Indices array
 39 | 
 40 |     // The FileVersion should always be "V3.2" for all GFF files that use the Generic File Format as
 41 |     // described in this document. If the FileVersion is different, then the application should abort reading the
 42 |     // GFF file.
 43 | 
 44 |     // The FileType is a programmer-defined 4-byte character string that identifies the content-type of the
 45 |     // GFF file. By convention, it is a 3-letter file extension in all-caps, followed by a space. For example,
 46 |     // "DLG ", "ITP ", etc. When opening a GFF file, the application should check the FileType to make
 47 |     // sure that the file being opened is of the expected type
 48 | };
 49 | 
 50 | struct GffStruct
 51 | {
 52 |     // In a GFF file, Struct Fields are stored differently from other fields. Whereas most Fields are stored in
 53 |     // the Field array, Structs are stored in the Struct Array.
 54 | 
 55 |     // The very first element in the Struct Array is the Top-Level Struct for the GFF file, and it "contains" all
 56 |     // the other Fields, Structs, and Lists. In this sense, the word "contain" refers to conceptual containment
 57 |     // (as in Section 2.1) rather than physical containment (as in Section 3). In other words, it does not imply
 58 |     // that all the other Fields are physically located inside the Top-Level Struct on disk (in fact, all Structs
 59 |     // have the same physical size on disk).
 60 | 
 61 |     // Since the Top-Level Struct is always present, every GFF file contains at least one element in the Struct
 62 |     // Array.
 63 | 
 64 |     // The Struct Array looks like this:
 65 | 
 66 |     // Programmer-defined integer ID
 67 |     std::uint32_t m_Type;
 68 | 
 69 |     // If Struct.FieldCount = 1, this is an index into the Field Array.
 70 |     // If Struct.FieldCount > 1, this is a byte offset into the Field Indices
 71 |     // array, where there is an array of DWORDs having a number of
 72 |     // elements equal to Struct.FieldCount. Each one of these DWORDs
 73 |     // is an index into the Field Array.
 74 |     std::uint32_t m_DataOrDataOffset;
 75 | 
 76 |     // Number of fields in this Struct.
 77 |     std::uint32_t m_FieldCount;
 78 | };
 79 | 
 80 | struct GffList
 81 | {
 82 |     // The List Indices Array contains a sequence of List elements packed end-to-end.
 83 |     // A List is an array of Structs, and being array, its length is variable. The format of a List is as shown
 84 |     // below:
 85 |     // std::uint32_t m_Size;
 86 |     // std::uint32_t m_Elements[m_Size];
 87 |     std::vector<std::uint32_t> m_Elements;
 88 | };
 89 | 
 90 | struct GffField
 91 | {
 92 |     // The Field Array contains all the Fields in the GFF file except for the Top-Level Struct.
 93 |     // Each Field contains the values listed in the table below. All of the values are DWORDs.
 94 | 
 95 |     enum class Type : std::uint32_t
 96 |     {
 97 |         // The Field Type specifies what data type the Field stores (recall the data types from Section 2.2). The
 98 |         // following table lists the values for each Field type. A datatype is considered complex if it would not fit
 99 |         // within a DWORD (4 bytes).
100 | 
101 |         // NOTE: We define u8/u16/etc which aren't technically part of the standard but make it much easier to understand
102 |         // what each type actually is.
103 |         BYTE, u8 = BYTE,
104 |         CHAR,
105 |         WORD, u16 = WORD,
106 |         SHORT, i16 = SHORT,
107 |         DWORD, u32 = DWORD,
108 |         INT, i32 = INT,
109 | 
110 |         // A DWORD64 is a 64-bit (8-byte) unsigned integer. As with all integer values in GFF, the least
111 |         // significant byte comes first, and the most significant byte is last.
112 |         DWORD64, u64 = DWORD64, // See *
113 | 
114 |         // An INT64 is a 64-bit (8-byte) signed integer. As with all integer values in GFF, the least significant
115 |         // byte comes first, and the most significant byte is last.
116 |         INT64, i64 = INT64, // See *
117 | 
118 |         FLOAT, f32 = FLOAT,
119 | 
120 |         // A DOUBLE is a double-precision floating point value, and takes up 8 bytes. It is stored in little-endian
121 |         // byte order, with the least significant byte first.
122 |         // (Both the FLOAT and DOUBLE data types conform to IEEE Standard 754-1985).
123 |         DOUBLE, f64 = DOUBLE, // See *
124 | 
125 |         // Refer to Type_CExoString below.
126 |         CExoString, // See *
127 | 
128 |         // Refer to Type_CResRef below.
129 |         ResRef, // See *
130 | 
131 |         // Refer to Type_CExoLocString below.
132 |         CExoLocString, // See *
133 | 
134 |         // Refer to Type_Void below.
135 |         VOID, Binary = VOID, // See *
136 | 
137 |         // Refer to Type_Struct below.
138 |         Struct, // See **
139 | 
140 |         // Refer to Type_List below.
141 |         List // See ***
142 |     };
143 | 
144 |     using Type_BYTE = std::uint8_t;
145 |     using Type_CHAR = char;
146 |     using Type_WORD = std::uint16_t;
147 |     using Type_SHORT = std::int16_t;
148 |     using Type_DWORD = std::uint32_t;
149 |     using Type_INT = std::int32_t;
150 |     using Type_DWORD64 = std::uint64_t;
151 |     using Type_INT64 = std::int64_t;
152 |     using Type_FLOAT = float;
153 |     using Type_DOUBLE = double;
154 | 
155 |     struct Type_CExoString
156 |     {
157 |         // A CExoString is a simple character string datatype. The figure below shows the layout of a CExoString:
158 |         // std::uint32_t m_Size;
159 |         // char m_String[m_Size];
160 |         std::string m_String;
161 | 
162 |         // A CExoString begins with a single DWORD (4-byte unsigned integer) which stores the string's Size. It
163 |         // specifies how many characters are in the string. This character-count does not include a null terminator.
164 |         // If we let N equal the number stored in Size, then the next N bytes after the Size are the characters that
165 |         // make up the string. There is no null terminator.
166 |     };
167 | 
168 |     struct Type_CResRef
169 |     {
170 |         // A CResRef is used to store the name of a file used by the game or toolset. These files may be located in
171 |         // the BIF files in the user's data folder, inside an Encapsulated Resource File (ERF, MOD, or HAK), or in
172 |         // the user's override folder. For efficiency and to reduce network bandwidth, A ResRef can only have up
173 |         // to 16 characters and is not null-terminated. ResRefs are also non-case-sensitive and stored in all-lowercase.
174 |         // The diagram below shows the structure of a CResRef stored in a GFF:
175 |         std::uint8_t m_Size;
176 |         char m_String[16];
177 | 
178 |         // The first byte is a Size, an unsigned value specifying the number of characters to follow. The Size is 16
179 |         // at most. The character string contains no null terminator.
180 |     };
181 | 
182 |     struct Type_CExoLocString
183 |     {
184 |         // A CExoLocString is a localized string. It can contain 0 or more CExoStrings, each one for a different
185 |         // language and possibly gender. For a list of language IDs, see Table 2.2b.
186 |         // The figure below shows the layout of a CExoLocString
187 | 
188 |         struct SubString
189 |         {
190 |             // A LocString SubString has almost the same format as a CExoString, but includes an additional String
191 |             // ID at the beginning
192 |             std::int32_t m_StringID;
193 |             // std::uint32_t m_Size;
194 |             // char m_String[m_Size];
195 |             std::string m_String;
196 | 
197 |             // The StringID stored in a GFF file does not match up exactly to the LanguageIDs shown in Table 2.2b.
198 |             // Instead, it is 2 times the Language ID, plus the Gender (0 for neutral or masculine, 1 for feminine).
199 |             // If we let N equal the number stored in StringLength, then the N bytes after the StringLength are the
200 |             // characters that make up the string. There is no null terminator.
201 |         };
202 | 
203 |         // A CExoLocString begins with a single DWORD (4-byte unsigned integer) which stores the total
204 |         // number of bytes in the CExoLocString, not including the first 4 size bytes.
205 | 
206 |         // The next 4 bytes are a DWORD containing the StringRef of the LocString. The StringRef is an index
207 |         // into the user's dialog.tlk file, which contains a list of almost all the localized text in the game and
208 |         // toolset. If the StringRef is -1 (ie., 0xFFFFFFFF), then the LocString does not reference dialog.tlk at all.
209 | 
210 |         // The 4 bytes after the StringRef comprise the StringCount, a DWORD that specifies how many
211 |         // SubStrings the LocString contains. The remainder of the LocString is a list of SubStrings.
212 | 
213 |         std::uint32_t m_TotalSize;
214 |         std::uint32_t m_StringRef;
215 |         // std::uint32_t m_StringCount;
216 |         // SubString m_SubStrings[m_StringCount];
217 |         std::vector<SubString> m_SubStrings;
218 |     };
219 | 
220 |     struct Type_VOID
221 |     {
222 |         // Void data is an arbitrary sequence of bytes to be interpreted by the application in a programmer-defined
223 |         // fashion. The format is shown below:
224 |         // std::uint32_t m_Size;
225 |         // std::byte m_Data[m_Size];
226 |         std::vector<std::byte> m_Data;
227 | 
228 |         // Size is a DWORD containing the number of bytes of data. The data itself is contained in the N bytes
229 |         // that follow, where N is equal to the Size value.
230 |     };
231 | 
232 |     // Unlike most of the complex Field data types, a Struct Field's data is located not in the Field Data Block,
233 |     // but in the Struct Array.
234 |     // Normally, a Field's DataOrDataOffset value would be a byte offset into the Field Data Block, but for a
235 |     // Struct, it is an index into the Struct Array.
236 |     // For information on the layout of a Struct, see Section 3.3, with particular attention to Table 3.3.
237 |     using Type_Struct = GffStruct;
238 | 
239 |     // Unlike most of the complex Field data types, a List Field's data is located not in the Field Data Block,
240 |     // but in the Field Indices Array.
241 |     // The starting address of a List is specified in its Field's DataOrDataOffset value as a byte offset into the
242 |     // Field Indices Array, at which is located a List element. Section 3.8 describes the structure a List
243 |     // element.
244 |     using Type_List = GffList;
245 | 
246 |     // Data type
247 |     Type m_Type;
248 | 
249 |     // Index into the Label Array
250 |     std::uint32_t m_LabelIndex;
251 | 
252 |     // If Field.Type is a simple data type (see table below), then this is
253 |     // the value actual of the field.
254 |     // If Field.Type is a complex data type (see table below), then this is
255 |     // a byte offset into the Field Data block.
256 |     std::uint32_t m_DataOrDataOffset;
257 | 
258 |     // Non-complex Field data is contained directly within the Field itself, in the DataOrDataOffset member.
259 | 
260 |     // If the data type is smaller than a DWORD, then the first bytes of the DataOrDataOffset DWORD, up to
261 |     // the size of the data type itself, contain the data value.
262 | 
263 |     // * If the Field data is complex, then the DataOrDataOffset value is equal to a byte offset from the
264 |     // beginning of the Field Data Block, pointing to the raw bytes that represent the complex data. The exact
265 |     // method of fetching the complex data depends on the actual Field Type, and is described in Section 4.
266 | 
267 |     // ** As a special case, if the Field Type is a Struct, then DataOrDataOffset is an index into the Struct Array
268 |     // instead of a byte offset into the Field Data Block.
269 | 
270 |     // *** As another special case, if the Field Type is a List, then DataOrDataOffset is a byte offset from the
271 |     // beginning of the List Indices Array, where there is a DWORD for the size of the array followed by an
272 |     // array of DWORDs. The elements of the array are offsets into the Struct Array. See Section 4.9 for
273 |     // details.
274 | };
275 | 
276 | struct GffLabel
277 | {
278 |     // A Label is a 16-CHAR array. Unused characters are nulls, but the label itself is non-null-terminated, so
279 |     // a 16-character label would use up all 16 CHARs with no null at the end.
280 | 
281 |     // The Label Array is a list of all the Labels used in a GFF file.
282 | 
283 |     // Note that a single Label may be referenced by more than one Field. When multiple Fields have Labels
284 |     // with the exact same text, they share the same Label element instead of each having their own copy.
285 |     // This sharing occurs regardless of what Struct the Field belongs to. All Labels in the Label Array should be
286 |     // unique.
287 | 
288 |     // Also, the Fields belonging to a Struct must all use different Labels. It is permissible, however, for Fields
289 |     // in two different Structs to use the same Label, regardless of whether one of those Structs is conceptually
290 |     // contained inside the other Struct.
291 |     char m_Label[16];
292 | };
293 | 
294 | // The Field Data block contains raw data for any Fields that have a complex Field Type, as described in
295 | // Section 3.4. The two exceptions to this rule are Struct and List Fields, which are not stored in the Field
296 | // Data Block.
297 | 
298 | // The FieldDataCount in the GFF header specifies the number of BYTEs contained in the Field Data
299 | // block.
300 | 
301 | // The data in the Field Data Block is laid out according to the type of Field that owns each byte of data.
302 | // See Section 4 for details
303 | using GffFieldData = std::byte;
304 | 
305 | // A Field Index is a DWORD containing the index of the associated Field within the Field array.
306 | // The Field Indices Array is an array of such DWORDs
307 | using GffFieldIndex = std::uint32_t;
308 | 
309 | //The List Indices Array contains a sequence of List elements packed end - to - end.
310 | // A List is an array of Structs, and being array, its length is variable.
311 | // The first DWORD is the Size of the List, and it specifies how many Struct elements the List contains.
312 | // There are Size DWORDS after that, each one an index into the Struct Array.
313 | using GffListIndex = std::byte;
314 | 
315 | struct Gff
316 | {
317 |     GffHeader m_Header;
318 |     std::vector<GffStruct> m_Structs;
319 |     std::vector<GffField> m_Fields;
320 |     std::vector<GffLabel> m_Labels;
321 |     std::vector<GffFieldData> m_FieldData;
322 |     std::vector<GffFieldIndex> m_FieldIndices;
323 |     std::vector<GffListIndex> m_ListIndices;
324 | 
325 |     // Constructs an Gff from a non-owning pointer.
326 |     static bool ReadFromBytes(std::byte const* bytes, Gff* out);
327 | 
328 |     // Constructs an Gff from a vector of bytes which we have taken ownership of.
329 |     static bool ReadFromByteVector(std::vector<std::byte>&& bytes, Gff* out);
330 | 
331 |     // Constructs an Gff from a file.
332 |     static bool ReadFromFile(char const* path, Gff* out);
333 | 
334 |     // Writes the raw Gff to disk.
335 |     bool WriteToFile(char const* path) const;
336 | 
337 |     // Below are functions to construct a type from the provided field.
338 |     GffField::Type_BYTE ConstructBYTE(GffField const& field) const;
339 |     GffField::Type_CHAR ConstructCHAR(GffField const& field) const;
340 |     GffField::Type_WORD ConstructWORD(GffField const& field) const;
341 |     GffField::Type_SHORT ConstructSHORT(GffField const& field) const;
342 |     GffField::Type_DWORD ConstructDWORD(GffField const& field) const;
343 |     GffField::Type_INT ConstructINT(GffField const& field) const;
344 |     GffField::Type_DWORD64 ConstructDWORD64(GffField const& field) const;
345 |     GffField::Type_INT64 ConstructINT64(GffField const& field) const;
346 |     GffField::Type_FLOAT ConstructFLOAT(GffField const& field) const;
347 |     GffField::Type_DOUBLE ConstructDOUBLE(GffField const& field) const;
348 |     GffField::Type_CExoString ConstructCExoString(GffField const& field) const;
349 |     GffField::Type_CResRef ConstructResRef(GffField const& field) const;
350 |     GffField::Type_CExoLocString ConstructCExoLocString(GffField const& field) const;
351 |     GffField::Type_VOID ConstructVOID(GffField const& field) const;
352 |     GffField::Type_Struct ConstructStruct(GffField const& field) const;
353 |     GffField::Type_List ConstructList(GffField const& field) const;
354 | 
355 | private:
356 |     bool ConstructInternal(std::byte const* bytes);
357 |     void ReadStructs(std::byte const* data);
358 |     void ReadFields(std::byte const* data);
359 |     void ReadLabels(std::byte const* data);
360 |     void ReadFieldData(std::byte const* data);
361 |     void ReadFieldIndices(std::byte const* data);
362 |     void ReadLists(std::byte const* data);
363 | };
364 | 
365 | }
366 | 


--------------------------------------------------------------------------------