From 906d785c73cb3644e8984fbfcbf5fcb8a1ebbc6f Mon Sep 17 00:00:00 2001
From: Zach Hilman <zachhilman@gmail.com>
Date: Fri, 27 Jul 2018 18:14:03 -0400
Subject: [PATCH] RomFS Extraction
---
src/core/CMakeLists.txt | 4 +
src/core/file_sys/content_archive.cpp | 19 +---
src/core/file_sys/romfs.cpp | 124 ++++++++++++++++++++++++++
src/core/file_sys/romfs.h | 35 ++++++++
src/core/file_sys/vfs.cpp | 23 +++++
src/core/file_sys/vfs.h | 20 +++++
src/core/file_sys/vfs_offset.cpp | 7 +-
src/core/file_sys/vfs_offset.h | 3 +-
src/core/file_sys/vfs_real.cpp | 6 ++
src/core/file_sys/vfs_real.h | 3 +-
src/core/file_sys/vfs_vector.cpp | 83 +++++++++++++++++
src/core/file_sys/vfs_vector.h | 44 +++++++++
12 files changed, 351 insertions(+), 20 deletions(-)
create mode 100644 src/core/file_sys/romfs.cpp
create mode 100644 src/core/file_sys/romfs.h
create mode 100644 src/core/file_sys/vfs_vector.cpp
create mode 100644 src/core/file_sys/vfs_vector.h
diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt
index b367c2a279..f389897d46 100644
--- a/src/core/CMakeLists.txt
+++ b/src/core/CMakeLists.txt
@@ -23,6 +23,8 @@ add_library(core STATIC
file_sys/partition_filesystem.h
file_sys/program_metadata.cpp
file_sys/program_metadata.h
+ file_sys/romfs.cpp
+ file_sys/romfs.h
file_sys/romfs_factory.cpp
file_sys/romfs_factory.h
file_sys/savedata_factory.cpp
@@ -35,6 +37,8 @@ add_library(core STATIC
file_sys/vfs_offset.h
file_sys/vfs_real.cpp
file_sys/vfs_real.h
+ file_sys/vfs_vector.cpp
+ file_sys/vfs_vector.h
frontend/emu_window.cpp
frontend/emu_window.h
frontend/framebuffer_layout.cpp
diff --git a/src/core/file_sys/content_archive.cpp b/src/core/file_sys/content_archive.cpp
index d6b20c0476..61cb0bbe30 100644
--- a/src/core/file_sys/content_archive.cpp
+++ b/src/core/file_sys/content_archive.cpp
@@ -9,6 +9,7 @@
#include "core/file_sys/content_archive.h"
#include "core/file_sys/vfs_offset.h"
#include "core/loader/loader.h"
+#include "romfs.h"
namespace FileSys {
@@ -46,21 +47,9 @@ struct PFS0Superblock {
};
static_assert(sizeof(PFS0Superblock) == 0x200, "PFS0Superblock has incorrect size.");
-struct IVFCLevel {
- u64_le offset;
- u64_le size;
- u32_le block_size;
- u32_le reserved;
-};
-static_assert(sizeof(IVFCLevel) == 0x18, "IVFCLevel has incorrect size.");
-
struct RomFSSuperblock {
NCASectionHeaderBlock header_block;
- u32_le magic;
- u32_le magic_number;
- INSERT_PADDING_BYTES(8);
- std::array<IVFCLevel, 6> levels;
- INSERT_PADDING_BYTES(64);
+ IVFCHeader ivfc;
};
static_assert(sizeof(RomFSSuperblock) == 0xE8, "RomFSSuperblock has incorrect size.");
@@ -92,8 +81,8 @@ NCA::NCA(VirtualFile file_) : file(std::move(file_)) {
const size_t romfs_offset =
header.section_tables[i].media_offset * MEDIA_OFFSET_MULTIPLIER +
- sb.levels[IVFC_MAX_LEVEL - 1].offset;
- const size_t romfs_size = sb.levels[IVFC_MAX_LEVEL - 1].size;
+ sb.ivfc.levels[IVFC_MAX_LEVEL - 1].offset;
+ const size_t romfs_size = sb.ivfc.levels[IVFC_MAX_LEVEL - 1].size;
files.emplace_back(std::make_shared<OffsetVfsFile>(file, romfs_size, romfs_offset));
romfs = files.back();
} else if (block.filesystem_type == NCASectionFilesystemType::PFS0) {
diff --git a/src/core/file_sys/romfs.cpp b/src/core/file_sys/romfs.cpp
new file mode 100644
index 0000000000..ff3ddb29ca
--- /dev/null
+++ b/src/core/file_sys/romfs.cpp
@@ -0,0 +1,124 @@
+// Copyright 2018 yuzu emulator team
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#include "common/common_types.h"
+#include "common/swap.h"
+#include "core/file_sys/romfs.h"
+#include "core/file_sys/vfs.h"
+#include "core/file_sys/vfs_offset.h"
+#include "core/file_sys/vfs_vector.h"
+
+namespace FileSys {
+
+constexpr u32 ROMFS_ENTRY_EMPTY = 0xFFFFFFFF;
+
+struct TableLocation {
+ u64_le offset;
+ u64_le size;
+};
+static_assert(sizeof(TableLocation) == 0x10, "TableLocation has incorrect size.");
+
+struct RomFSHeader {
+ u64_le header_size;
+ TableLocation directory_hash;
+ TableLocation directory_meta;
+ TableLocation file_hash;
+ TableLocation file_meta;
+ u64_le data_offset;
+};
+static_assert(sizeof(RomFSHeader) == 0x50, "RomFSHeader has incorrect size.");
+
+struct DirectoryEntry {
+ u32_le sibling;
+ u32_le child_dir;
+ u32_le child_file;
+ u32_le hash;
+ u32_le name_length;
+};
+static_assert(sizeof(DirectoryEntry) == 0x14, "DirectoryEntry has incorrect size.");
+
+struct FileEntry {
+ u32_le parent;
+ u32_le sibling;
+ u64_le offset;
+ u64_le size;
+ u32_le hash;
+ u32_le name_length;
+};
+static_assert(sizeof(FileEntry) == 0x20, "FileEntry has incorrect size.");
+
+template <typename Entry>
+static std::pair<Entry, std::string> GetEntry(const VirtualFile& file, size_t offset) {
+ Entry entry{};
+ if (file->ReadObject(&entry, offset) != sizeof(Entry))
+ return {};
+ std::string string(entry.name_length, '\0');
+ if (file->ReadArray(&string[0], string.size(), offset + sizeof(Entry)) != string.size())
+ return {};
+ return {entry, string};
+}
+
+void ProcessFile(VirtualFile file, size_t file_offset, size_t data_offset, u32 this_file_offset,
+ std::shared_ptr<VectorVfsDirectory> parent) {
+ while (true) {
+ auto entry = GetEntry<FileEntry>(file, file_offset + this_file_offset);
+
+ parent->AddFile(std::make_shared<OffsetVfsFile>(
+ file, entry.first.size, entry.first.offset + data_offset, entry.second, parent));
+
+ if (entry.first.sibling == ROMFS_ENTRY_EMPTY)
+ break;
+
+ this_file_offset = entry.first.sibling;
+ }
+}
+
+void ProcessDirectory(VirtualFile file, size_t dir_offset, size_t file_offset, size_t data_offset,
+ u32 this_dir_offset, std::shared_ptr<VectorVfsDirectory> parent) {
+ while (true) {
+ auto entry = GetEntry<DirectoryEntry>(file, dir_offset + this_dir_offset);
+ auto current = std::make_shared<VectorVfsDirectory>(
+ std::vector<VirtualFile>{}, std::vector<VirtualDir>{}, parent, entry.second);
+
+ if (entry.first.child_file != ROMFS_ENTRY_EMPTY) {
+ ProcessFile(file, file_offset, data_offset, entry.first.child_file, current);
+ }
+
+ if (entry.first.child_dir != ROMFS_ENTRY_EMPTY) {
+ ProcessDirectory(file, dir_offset, file_offset, data_offset, entry.first.child_dir,
+ current);
+ }
+
+ parent->AddDirectory(current);
+ if (entry.first.sibling == ROMFS_ENTRY_EMPTY)
+ break;
+ this_dir_offset = entry.first.sibling;
+ }
+}
+
+VirtualDir ExtractRomFS(VirtualFile file) {
+ RomFSHeader header{};
+ if (file->ReadObject(&header) != sizeof(RomFSHeader))
+ return nullptr;
+
+ if (header.header_size != sizeof(RomFSHeader))
+ return nullptr;
+
+ const u64 file_offset = header.file_meta.offset;
+ const u64 dir_offset = header.directory_meta.offset + 4;
+
+ const auto root =
+ std::make_shared<VectorVfsDirectory>(std::vector<VirtualFile>{}, std::vector<VirtualDir>{},
+ file->GetContainingDirectory(), file->GetName());
+
+ ProcessDirectory(file, dir_offset, file_offset, header.data_offset, 0, root);
+
+ VirtualDir out = std::move(root);
+
+ while (out->GetSubdirectory("") != nullptr)
+ out = out->GetSubdirectory("");
+
+ return out;
+}
+} // namespace FileSys
diff --git a/src/core/file_sys/romfs.h b/src/core/file_sys/romfs.h
new file mode 100644
index 0000000000..03a876d229
--- /dev/null
+++ b/src/core/file_sys/romfs.h
@@ -0,0 +1,35 @@
+// Copyright 2018 yuzu emulator team
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#pragma once
+
+#include <array>
+#include "common/common_funcs.h"
+#include "common/swap.h"
+#include "core/file_sys/vfs.h"
+
+namespace FileSys {
+
+struct IVFCLevel {
+ u64_le offset;
+ u64_le size;
+ u32_le block_size;
+ u32_le reserved;
+};
+static_assert(sizeof(IVFCLevel) == 0x18, "IVFCLevel has incorrect size.");
+
+struct IVFCHeader {
+ u32_le magic;
+ u32_le magic_number;
+ INSERT_PADDING_BYTES(8);
+ std::array<IVFCLevel, 6> levels;
+ INSERT_PADDING_BYTES(64);
+};
+static_assert(sizeof(IVFCHeader) == 0xE0, "IVFCHeader has incorrect size.");
+
+// Converts a RomFS binary blob to VFS Filesystem
+// Returns nullptr on failure
+VirtualDir ExtractRomFS(VirtualFile file);
+
+} // namespace FileSys
diff --git a/src/core/file_sys/vfs.cpp b/src/core/file_sys/vfs.cpp
index b99a4fd5be..84a6a73978 100644
--- a/src/core/file_sys/vfs.cpp
+++ b/src/core/file_sys/vfs.cpp
@@ -46,6 +46,13 @@ size_t VfsFile::WriteBytes(const std::vector<u8>& data, size_t offset) {
return Write(data.data(), data.size(), offset);
}
+std::string VfsFile::GetFullPath() const {
+ if (GetContainingDirectory() == nullptr)
+ return "/" + GetName();
+
+ return GetContainingDirectory()->GetFullPath() + "/" + GetName();
+}
+
std::shared_ptr<VfsFile> VfsDirectory::GetFileRelative(std::string_view path) const {
auto vec = FileUtil::SplitPathComponents(path);
vec.erase(std::remove_if(vec.begin(), vec.end(), [](const auto& str) { return str.empty(); }),
@@ -243,6 +250,13 @@ bool VfsDirectory::Copy(std::string_view src, std::string_view dest) {
return f2->WriteBytes(f1->ReadAllBytes()) == f1->GetSize();
}
+std::string VfsDirectory::GetFullPath() const {
+ if (IsRoot())
+ return GetName();
+
+ return GetParentDirectory()->GetFullPath() + "/" + GetName();
+}
+
bool ReadOnlyVfsDirectory::IsWritable() const {
return false;
}
@@ -270,4 +284,13 @@ bool ReadOnlyVfsDirectory::DeleteFile(std::string_view name) {
bool ReadOnlyVfsDirectory::Rename(std::string_view name) {
return false;
}
+
+bool VfsRawCopy(VirtualFile src, VirtualFile dest) {
+ if (src == nullptr || dest == nullptr)
+ return false;
+ if (!dest->Resize(src->GetSize()))
+ return false;
+ std::vector<u8> data = src->ReadAllBytes();
+ return dest->WriteBytes(data, 0) == data.size();
+}
} // namespace FileSys
diff --git a/src/core/file_sys/vfs.h b/src/core/file_sys/vfs.h
index 4a13b8378e..cf871edd67 100644
--- a/src/core/file_sys/vfs.h
+++ b/src/core/file_sys/vfs.h
@@ -113,6 +113,9 @@ struct VfsFile : NonCopyable {
// Renames the file to name. Returns whether or not the operation was successsful.
virtual bool Rename(std::string_view name) = 0;
+
+ // Returns the full path of this file as a string, recursively
+ virtual std::string GetFullPath() const;
};
// A class representing a directory in an abstract filesystem.
@@ -213,6 +216,17 @@ struct VfsDirectory : NonCopyable {
return ReplaceFileWithSubdirectory(file_p, std::make_shared<Directory>(file_p));
}
+ bool InterpretAsDirectory(const std::function<VirtualDir(VirtualFile)>& function,
+ const std::string& file) {
+ auto file_p = GetFile(file);
+ if (file_p == nullptr)
+ return false;
+ return ReplaceFileWithSubdirectory(file_p, function(file_p));
+ }
+
+ // Returns the full path of this directory as a string, recursively
+ virtual std::string GetFullPath() const;
+
protected:
// Backend for InterpretAsDirectory.
// Removes all references to file and adds a reference to dir in the directory's implementation.
@@ -230,4 +244,10 @@ struct ReadOnlyVfsDirectory : public VfsDirectory {
bool DeleteFile(std::string_view name) override;
bool Rename(std::string_view name) override;
};
+
+// A method that copies the raw data between two different implementations of VirtualFile. If you
+// are using the same implementation, it is probably better to use the Copy method in the parent
+// directory of src/dest.
+bool VfsRawCopy(VirtualFile src, VirtualFile dest);
+
} // namespace FileSys
diff --git a/src/core/file_sys/vfs_offset.cpp b/src/core/file_sys/vfs_offset.cpp
index a40331cefd..847cde2f5c 100644
--- a/src/core/file_sys/vfs_offset.cpp
+++ b/src/core/file_sys/vfs_offset.cpp
@@ -10,8 +10,9 @@
namespace FileSys {
OffsetVfsFile::OffsetVfsFile(std::shared_ptr<VfsFile> file_, size_t size_, size_t offset_,
- std::string name_)
- : file(std::move(file_)), offset(offset_), size(size_), name(std::move(name_)) {}
+ std::string name_, VirtualDir parent_)
+ : file(file_), offset(offset_), size(size_), name(std::move(name_)),
+ parent(parent_ == nullptr ? file->GetContainingDirectory() : std::move(parent_)) {}
std::string OffsetVfsFile::GetName() const {
return name.empty() ? file->GetName() : name;
@@ -35,7 +36,7 @@ bool OffsetVfsFile::Resize(size_t new_size) {
}
std::shared_ptr<VfsDirectory> OffsetVfsFile::GetContainingDirectory() const {
- return file->GetContainingDirectory();
+ return parent;
}
bool OffsetVfsFile::IsWritable() const {
diff --git a/src/core/file_sys/vfs_offset.h b/src/core/file_sys/vfs_offset.h
index 4f471e3ba4..235970dc5e 100644
--- a/src/core/file_sys/vfs_offset.h
+++ b/src/core/file_sys/vfs_offset.h
@@ -17,7 +17,7 @@ namespace FileSys {
// the size of this wrapper.
struct OffsetVfsFile : public VfsFile {
OffsetVfsFile(std::shared_ptr<VfsFile> file, size_t size, size_t offset = 0,
- std::string new_name = "");
+ std::string new_name = "", VirtualDir new_parent = nullptr);
std::string GetName() const override;
size_t GetSize() const override;
@@ -44,6 +44,7 @@ private:
size_t offset;
size_t size;
std::string name;
+ VirtualDir parent;
};
} // namespace FileSys
diff --git a/src/core/file_sys/vfs_real.cpp b/src/core/file_sys/vfs_real.cpp
index 9ce2e1efa0..82d54da4a3 100644
--- a/src/core/file_sys/vfs_real.cpp
+++ b/src/core/file_sys/vfs_real.cpp
@@ -195,6 +195,12 @@ bool RealVfsDirectory::Rename(std::string_view name) {
return FileUtil::Rename(path, new_name);
}
+std::string RealVfsDirectory::GetFullPath() const {
+ auto out = path;
+ std::replace(out.begin(), out.end(), '\\', '/');
+ return out;
+}
+
bool RealVfsDirectory::ReplaceFileWithSubdirectory(VirtualFile file, VirtualDir dir) {
const auto iter = std::find(files.begin(), files.end(), file);
if (iter == files.end())
diff --git a/src/core/file_sys/vfs_real.h b/src/core/file_sys/vfs_real.h
index 2151211c93..243d585767 100644
--- a/src/core/file_sys/vfs_real.h
+++ b/src/core/file_sys/vfs_real.h
@@ -41,7 +41,7 @@ private:
// An implementation of VfsDirectory that represents a directory on the user's computer.
struct RealVfsDirectory : public VfsDirectory {
- RealVfsDirectory(const std::string& path, Mode perms);
+ RealVfsDirectory(const std::string& path, Mode perms = Mode::Read);
std::vector<std::shared_ptr<VfsFile>> GetFiles() const override;
std::vector<std::shared_ptr<VfsDirectory>> GetSubdirectories() const override;
@@ -54,6 +54,7 @@ struct RealVfsDirectory : public VfsDirectory {
bool DeleteSubdirectory(std::string_view name) override;
bool DeleteFile(std::string_view name) override;
bool Rename(std::string_view name) override;
+ std::string GetFullPath() const override;
protected:
bool ReplaceFileWithSubdirectory(VirtualFile file, VirtualDir dir) override;
diff --git a/src/core/file_sys/vfs_vector.cpp b/src/core/file_sys/vfs_vector.cpp
new file mode 100644
index 0000000000..4c6337e3a3
--- /dev/null
+++ b/src/core/file_sys/vfs_vector.cpp
@@ -0,0 +1,83 @@
+// Copyright 2018 yuzu emulator team
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#include <algorithm>
+#include "core/file_sys/vfs_vector.h"
+
+namespace FileSys {
+VectorVfsDirectory::VectorVfsDirectory(std::vector<VirtualFile> files_,
+ std::vector<VirtualDir> dirs_, VirtualDir parent_,
+ std::string name_)
+ : files(std::move(files_)), dirs(std::move(dirs_)), parent(std::move(parent_)),
+ name(std::move(name_)) {}
+
+std::vector<std::shared_ptr<VfsFile>> VectorVfsDirectory::GetFiles() const {
+ return files;
+}
+
+std::vector<std::shared_ptr<VfsDirectory>> VectorVfsDirectory::GetSubdirectories() const {
+ return dirs;
+}
+
+bool VectorVfsDirectory::IsWritable() const {
+ return false;
+}
+
+bool VectorVfsDirectory::IsReadable() const {
+ return true;
+}
+
+std::string VectorVfsDirectory::GetName() const {
+ return name;
+}
+std::shared_ptr<VfsDirectory> VectorVfsDirectory::GetParentDirectory() const {
+ return parent;
+}
+
+template <typename T>
+static bool FindAndRemoveVectorElement(std::vector<T>& vec, std::string_view name) {
+ auto iter = std::find_if(vec.begin(), vec.end(), [name](T e) { return e->GetName() == name; });
+ if (iter == vec.end())
+ return false;
+ auto old_size = vec.size();
+ vec.erase(iter);
+ return true;
+}
+
+bool VectorVfsDirectory::DeleteSubdirectory(std::string_view name) {
+ return FindAndRemoveVectorElement(dirs, name);
+}
+
+bool VectorVfsDirectory::DeleteFile(std::string_view name) {
+ return FindAndRemoveVectorElement(files, name);
+}
+
+bool VectorVfsDirectory::Rename(std::string_view name_) {
+ name = name_;
+ return true;
+}
+
+std::shared_ptr<VfsDirectory> VectorVfsDirectory::CreateSubdirectory(std::string_view name) {
+ return nullptr;
+}
+
+std::shared_ptr<VfsFile> VectorVfsDirectory::CreateFile(std::string_view name) {
+ return nullptr;
+}
+
+void VectorVfsDirectory::AddFile(VirtualFile file) {
+ files.push_back(std::move(file));
+}
+
+void VectorVfsDirectory::AddDirectory(VirtualDir dir) {
+ dirs.push_back(std::move(dir));
+}
+
+bool VectorVfsDirectory::ReplaceFileWithSubdirectory(VirtualFile file, VirtualDir dir) {
+ if (!DeleteFile(file->GetName()))
+ return false;
+ dirs.emplace_back(dir);
+ return true;
+}
+} // namespace FileSys
diff --git a/src/core/file_sys/vfs_vector.h b/src/core/file_sys/vfs_vector.h
new file mode 100644
index 0000000000..ba469647bf
--- /dev/null
+++ b/src/core/file_sys/vfs_vector.h
@@ -0,0 +1,44 @@
+// Copyright 2018 yuzu emulator team
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#pragma once
+
+#include "core/file_sys/vfs.h"
+
+namespace FileSys {
+
+// An implementation of VfsDirectory that maintains two vectors for subdirectories and files.
+// Vector data is supplied upon construction.
+struct VectorVfsDirectory : public VfsDirectory {
+ explicit VectorVfsDirectory(std::vector<VirtualFile> files = {},
+ std::vector<VirtualDir> dirs = {}, VirtualDir parent = nullptr,
+ std::string name = "");
+
+ std::vector<std::shared_ptr<VfsFile>> GetFiles() const override;
+ std::vector<std::shared_ptr<VfsDirectory>> GetSubdirectories() const override;
+ bool IsWritable() const override;
+ bool IsReadable() const override;
+ std::string GetName() const override;
+ std::shared_ptr<VfsDirectory> GetParentDirectory() const override;
+ bool DeleteSubdirectory(std::string_view name) override;
+ bool DeleteFile(std::string_view name) override;
+ bool Rename(std::string_view name) override;
+ std::shared_ptr<VfsDirectory> CreateSubdirectory(std::string_view name) override;
+ std::shared_ptr<VfsFile> CreateFile(std::string_view name) override;
+
+ virtual void AddFile(VirtualFile file);
+ virtual void AddDirectory(VirtualDir dir);
+
+protected:
+ bool ReplaceFileWithSubdirectory(VirtualFile file, VirtualDir dir) override;
+
+private:
+ std::vector<VirtualFile> files;
+ std::vector<VirtualDir> dirs;
+
+ VirtualDir parent;
+ std::string name;
+};
+
+} // namespace FileSys