Merge pull request #5036 from leoetlino/bps

file_sys: Add support for BPS patching

src/core/CMakeLists.txt

@@ -74,6 +74,8 @@ add_library(core STATIC
     file_sys/ivfc_archive.h
     file_sys/ncch_container.cpp
     file_sys/ncch_container.h
+    file_sys/patch.cpp
+    file_sys/patch.h
     file_sys/path_parser.cpp
     file_sys/path_parser.h
     file_sys/romfs_reader.cpp

src/core/file_sys/ncch_container.cpp

@@ -12,6 +12,7 @@
 #include "common/logging/log.h"
 #include "core/core.h"
 #include "core/file_sys/ncch_container.h"
+#include "core/file_sys/patch.h"
 #include "core/file_sys/seed_db.h"
 #include "core/hw/aes/key.h"
 #include "core/loader/loader.h"
@@ -24,53 +25,6 @@ namespace FileSys {
 static const int kMaxSections = 8;   ///< Maximum number of sections (files) in an ExeFs
 static const int kBlockSize = 0x200; ///< Size of ExeFS blocks (in bytes)
 
-/**
- * Attempts to patch a buffer using an IPS
- * @param ips Vector of the patches to apply
- * @param buffer Vector to patch data into
- */
-static void ApplyIPS(std::vector<u8>& ips, std::vector<u8>& buffer) {
-    u32 cursor = 5;
-    u32 patch_length = ips.size() - 3;
-    std::string ips_header(ips.begin(), ips.begin() + 5);
-
-    if (ips_header != "PATCH") {
-        LOG_INFO(Service_FS, "Attempted to load invalid IPS");
-        return;
-    }
-
-    while (cursor < patch_length) {
-        std::string eof_check(ips.begin() + cursor, ips.begin() + cursor + 3);
-
-        if (eof_check == "EOF")
-            return;
-
-        u32 offset = ips[cursor] << 16 | ips[cursor + 1] << 8 | ips[cursor + 2];
-        std::size_t length = ips[cursor + 3] << 8 | ips[cursor + 4];
-
-        // check for an rle record
-        if (length == 0) {
-            length = ips[cursor + 5] << 8 | ips[cursor + 6];
-
-            if (buffer.size() < offset + length)
-                return;
-
-            for (u32 i = 0; i < length; ++i)
-                buffer[offset + i] = ips[cursor + 7];
-
-            cursor += 8;
-
-            continue;
-        }
-
-        if (buffer.size() < offset + length)
-            return;
-
-        std::memcpy(&buffer[offset], &ips[cursor + 5], length);
-        cursor += length + 5;
-    }
-}
-
 /**
  * Get the decompressed size of an LZSS compressed ExeFS file
  * @param buffer Buffer of compressed file
@@ -553,20 +507,32 @@ Loader::ResultStatus NCCHContainer::LoadSectionExeFS(const char* name, std::vect
     return Loader::ResultStatus::ErrorNotUsed;
 }
 
-bool NCCHContainer::ApplyIPSPatch(std::vector<u8>& code) const {
+Loader::ResultStatus NCCHContainer::ApplyCodePatch(std::vector<u8>& code) const {
-    const std::string override_ips = filepath + ".exefsdir/code.ips";
+    struct PatchLocation {
+        std::string path;
+        bool (*patch_fn)(const std::vector<u8>& patch, std::vector<u8>& code);
+    };
+    const std::array<PatchLocation, 2> patch_paths{{
+        {filepath + ".exefsdir/code.ips", Patch::ApplyIpsPatch},
+        {filepath + ".exefsdir/code.bps", Patch::ApplyBpsPatch},
+    }};
 
-    FileUtil::IOFile ips_file{override_ips, "rb"};
+    for (const PatchLocation& info : patch_paths) {
-    if (!ips_file)
+        FileUtil::IOFile file{info.path, "rb"};
-        return false;
+        if (!file)
+            continue;
 
-    std::vector<u8> ips(ips_file.GetSize());
+        std::vector<u8> patch(file.GetSize());
-    if (ips_file.ReadBytes(ips.data(), ips.size()) != ips.size())
+        if (file.ReadBytes(patch.data(), patch.size()) != patch.size())
-        return false;
+            return Loader::ResultStatus::Error;
 
-    LOG_INFO(Service_FS, "File {} patching code.bin", override_ips);
+        LOG_INFO(Service_FS, "File {} patching code.bin", info.path);
-    ApplyIPS(ips, code);
+        if (!info.patch_fn(patch, code))
-    return true;
+            return Loader::ResultStatus::Error;
+
+        return Loader::ResultStatus::Success;
+    }
+    return Loader::ResultStatus::ErrorNotUsed;
 }
 
 Loader::ResultStatus NCCHContainer::LoadOverrideExeFSSection(const char* name,

src/core/file_sys/ncch_container.h

@@ -272,11 +272,11 @@ public:
     Loader::ResultStatus ReadExtdataId(u64& extdata_id);
 
     /**
-     * Apply an IPS patch for .code (if it exists).
+     * Apply a patch for .code (if it exists).
      * This should only be called after allocating .bss.
-     * @return bool true if a patch was applied, false otherwise
+     * @return ResultStatus success if a patch was applied, ErrorNotUsed if no patch was found
      */
-    bool ApplyIPSPatch(std::vector<u8>& code) const;
+    Loader::ResultStatus ApplyCodePatch(std::vector<u8>& code) const;
 
     /**
      * Checks whether the NCCH container contains an ExeFS

src/core/file_sys/patch.cpp

@@ -0,0 +1,268 @@
+// Copyright 2019 Citra Emulator Project
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#include <array>
+#include <cstring>
+#include <optional>
+#include <string>
+#include <string_view>
+#include <type_traits>
+#include <boost/crc.hpp>
+#include "common/logging/log.h"
+#include "core/file_sys/patch.h"
+
+namespace FileSys::Patch {
+
+bool ApplyIpsPatch(const std::vector<u8>& ips, std::vector<u8>& buffer) {
+    u32 cursor = 5;
+    u32 patch_length = ips.size() - 3;
+    std::string ips_header(ips.begin(), ips.begin() + 5);
+
+    if (ips_header != "PATCH") {
+        LOG_INFO(Service_FS, "Attempted to load invalid IPS");
+        return false;
+    }
+
+    while (cursor < patch_length) {
+        std::string eof_check(ips.begin() + cursor, ips.begin() + cursor + 3);
+
+        if (eof_check == "EOF")
+            return false;
+
+        u32 offset = ips[cursor] << 16 | ips[cursor + 1] << 8 | ips[cursor + 2];
+        std::size_t length = ips[cursor + 3] << 8 | ips[cursor + 4];
+
+        // check for an rle record
+        if (length == 0) {
+            length = ips[cursor + 5] << 8 | ips[cursor + 6];
+
+            if (buffer.size() < offset + length)
+                return false;
+
+            for (u32 i = 0; i < length; ++i)
+                buffer[offset + i] = ips[cursor + 7];
+
+            cursor += 8;
+
+            continue;
+        }
+
+        if (buffer.size() < offset + length)
+            return false;
+
+        std::memcpy(&buffer[offset], &ips[cursor + 5], length);
+        cursor += length + 5;
+    }
+    return true;
+}
+
+namespace Bps {
+
+// The BPS format uses variable length encoding for all integers.
+// Realistically uint32s are more than enough for code patching.
+using Number = u32;
+
+constexpr std::size_t FooterSize = 12;
+
+// The BPS format uses CRC32 checksums.
+static u32 crc32(const u8* data, std::size_t size) {
+    boost::crc_32_type result;
+    result.process_bytes(data, size);
+    return result.checksum();
+}
+
+// Utility class to make keeping track of offsets and bound checks less error prone.
+template <typename T>
+class Stream {
+public:
+    Stream(T* ptr, std::size_t size) : m_ptr{ptr}, m_size{size} {}
+
+    bool Read(void* buffer, std::size_t length) {
+        if (m_offset + length > m_size)
+            return false;
+        std::memcpy(buffer, m_ptr + m_offset, length);
+        m_offset += length;
+        return true;
+    }
+
+    template <typename OtherType>
+    bool CopyFrom(Stream<OtherType>& other, std::size_t length) {
+        if (m_offset + length > m_size)
+            return false;
+        if (!other.Read(m_ptr + m_offset, length))
+            return false;
+        m_offset += length;
+        return true;
+    }
+
+    template <typename ValueType>
+    std::optional<ValueType> Read() {
+        static_assert(std::is_pod_v<ValueType>);
+        ValueType val{};
+        if (!Read(&val, sizeof(val)))
+            return std::nullopt;
+        return val;
+    }
+
+    Number ReadNumber() {
+        Number data = 0, shift = 1;
+        std::optional<u8> x;
+        while ((x = Read<u8>())) {
+            data += (*x & 0x7f) * shift;
+            if (*x & 0x80)
+                break;
+            shift <<= 7;
+            data += shift;
+        }
+        return data;
+    }
+
+    auto data() const {
+        return m_ptr;
+    }
+
+    std::size_t size() const {
+        return m_size;
+    }
+
+    std::size_t Tell() const {
+        return m_offset;
+    }
+
+    bool Seek(size_t offset) {
+        if (offset > m_size)
+            return false;
+        m_offset = offset;
+        return true;
+    }
+
+private:
+    T* m_ptr = nullptr;
+    std::size_t m_size = 0;
+    std::size_t m_offset = 0;
+};
+
+class PatchApplier {
+public:
+    PatchApplier(Stream<const u8> source, Stream<u8> target, Stream<const u8> patch)
+        : m_source{source}, m_target{target}, m_patch{patch} {}
+
+    bool Apply() {
+        const auto magic = *m_patch.Read<std::array<char, 4>>();
+        if (std::string_view(magic.data(), magic.size()) != "BPS1") {
+            LOG_ERROR(Service_FS, "Invalid BPS magic");
+            return false;
+        }
+
+        const Bps::Number source_size = m_patch.ReadNumber();
+        const Bps::Number target_size = m_patch.ReadNumber();
+        const Bps::Number metadata_size = m_patch.ReadNumber();
+        if (source_size > m_source.size() || target_size > m_target.size() || metadata_size != 0) {
+            LOG_ERROR(Service_FS, "Invalid sizes");
+            return false;
+        }
+
+        const std::size_t command_start_offset = m_patch.Tell();
+        const std::size_t command_end_offset = m_patch.size() - FooterSize;
+        m_patch.Seek(command_end_offset);
+        const u32 source_crc32 = *m_patch.Read<u32>();
+        const u32 target_crc32 = *m_patch.Read<u32>();
+        m_patch.Seek(command_start_offset);
+
+        if (crc32(m_source.data(), source_size) != source_crc32) {
+            LOG_ERROR(Service_FS, "Unexpected source hash");
+            return false;
+        }
+
+        // Process all patch commands.
+        std::memset(m_target.data(), 0, m_target.size());
+        while (m_patch.Tell() < command_end_offset) {
+            if (!HandleCommand())
+                return false;
+        }
+
+        if (crc32(m_target.data(), target_size) != target_crc32) {
+            LOG_ERROR(Service_FS, "Unexpected target hash");
+            return false;
+        }
+
+        return true;
+    }
+
+private:
+    bool HandleCommand() {
+        const std::size_t offset = m_patch.Tell();
+        const Number data = m_patch.ReadNumber();
+        const Number command = data & 3;
+        const Number length = (data >> 2) + 1;
+
+        const bool ok = [&] {
+            switch (command) {
+            case 0:
+                return SourceRead(length);
+            case 1:
+                return TargetRead(length);
+            case 2:
+                return SourceCopy(length);
+            case 3:
+                return TargetCopy(length);
+            default:
+                return false;
+            }
+        }();
+        if (!ok)
+            LOG_ERROR(Service_FS, "Failed to process command {} at 0x{:x}", command, offset);
+        return ok;
+    }
+
+    bool SourceRead(Number length) {
+        return m_source.Seek(m_target.Tell()) && m_target.CopyFrom(m_source, length);
+    }
+
+    bool TargetRead(Number length) {
+        return m_target.CopyFrom(m_patch, length);
+    }
+
+    bool SourceCopy(Number length) {
+        const Number data = m_patch.ReadNumber();
+        m_source_relative_offset += (data & 1 ? -1 : +1) * int(data >> 1);
+        if (!m_source.Seek(m_source_relative_offset) || !m_target.CopyFrom(m_source, length))
+            return false;
+        m_source_relative_offset += length;
+        return true;
+    }
+
+    bool TargetCopy(Number length) {
+        const Number data = m_patch.ReadNumber();
+        m_target_relative_offset += (data & 1 ? -1 : +1) * int(data >> 1);
+        if (m_target.Tell() + length > m_target.size())
+            return false;
+        if (m_target_relative_offset + length > m_target.size())
+            return false;
+        // Byte by byte copy.
+        for (size_t i = 0; i < length; ++i)
+            m_target.data()[m_target.Tell() + i] = m_target.data()[m_target_relative_offset++];
+        m_target.Seek(m_target.Tell() + length);
+        return true;
+    }
+
+    std::size_t m_source_relative_offset = 0;
+    std::size_t m_target_relative_offset = 0;
+    Stream<const u8> m_source;
+    Stream<u8> m_target;
+    Stream<const u8> m_patch;
+};
+
+} // namespace Bps
+
+bool ApplyBpsPatch(const std::vector<u8>& patch, std::vector<u8>& buffer) {
+    const std::vector<u8> source = buffer;
+    Bps::Stream source_stream{source.data(), source.size()};
+    Bps::Stream target_stream{buffer.data(), buffer.size()};
+    Bps::Stream patch_stream{patch.data(), patch.size()};
+    Bps::PatchApplier applier{source_stream, target_stream, patch_stream};
+    return applier.Apply();
+}
+
+} // namespace FileSys::Patch

src/core/file_sys/patch.h

@@ -0,0 +1,17 @@
+// Copyright 2019 Citra Emulator Project
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#pragma once
+
+#include <vector>
+
+#include "common/common_types.h"
+
+namespace FileSys::Patch {
+
+bool ApplyIpsPatch(const std::vector<u8>& patch, std::vector<u8>& buffer);
+
+bool ApplyBpsPatch(const std::vector<u8>& patch, std::vector<u8>& buffer);
+
+} // namespace FileSys::Patch

src/core/loader/ncch.cpp

@@ -100,8 +100,10 @@ ResultStatus AppLoader_NCCH::LoadExec(std::shared_ptr<Kernel::Process>& process)
             overlay_ncch->exheader_header.codeset_info.data.num_max_pages * Memory::PAGE_SIZE +
             bss_page_size;
 
-        // Apply any IPS patch now that the entire codeset (including .bss) has been allocated
+        // Apply patches now that the entire codeset (including .bss) has been allocated
-        overlay_ncch->ApplyIPSPatch(code);
+        const ResultStatus patch_result = overlay_ncch->ApplyCodePatch(code);
+        if (patch_result != ResultStatus::Success && patch_result != ResultStatus::ErrorNotUsed)
+            return patch_result;
 
         codeset->entrypoint = codeset->CodeSegment().addr;
         codeset->memory = std::move(code);