Updated config to support compressed rom

Otr extract 2

CMakeLists.txt

@@ -18,6 +18,8 @@ set(CMAKE_C_STANDARD 11 CACHE STRING "The C standard to use")
 
 # Add a custom module path to locate additional CMake modules
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/modules/")
+set(YAML_CPP_STATIC_DEFINE ON)
+add_compile_definitions(YAML_CPP_STATIC_DEFINE)
 
 if (WIN32)
 include(cmake/automate-vcpkg.cmake)
@@ -25,6 +27,10 @@ set(VCPKG_TRIPLET x64-windows-static)
 set(VCPKG_TARGET_TRIPLET x64-windows-static)
 vcpkg_bootstrap()
 vcpkg_install_packages(zlib bzip2 libzip libpng sdl2 glew glfw3 nlohmann-json tinyxml2 spdlog)
+if(NOT EXISTS ${CMAKE_BINARY_DIR}/config.yml)
+	execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different "${CMAKE_SOURCE_DIR}/config.yml" "${CMAKE_BINARY_DIR}/config.yml")
+	execute_process(COMMAND ${CMAKE_COMMAND} -E copy_directory "${CMAKE_SOURCE_DIR}/assets/" "${CMAKE_BINARY_DIR}/assets/")
+endif()
 endif()
 
 if (MSVC)
@@ -233,8 +239,7 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Windows")
 			">"
 			"INCLUDE_GAME_PRINTF;"
 			"UNICODE;"
-			"_UNICODE"
+			"_UNICODE;"
-			STORMLIB_NO_AUTO_LINK
 			"_CRT_SECURE_NO_WARNINGS;"
             "_SILENCE_ALL_MS_EXT_DEPRECATION_WARNINGS;"
 		)
@@ -252,11 +257,12 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Windows")
 			"INCLUDE_GAME_PRINTF;"
 			"WIN32;"
 			"UNICODE;"
-			"_UNICODE"
+			"_UNICODE;"
-			STORMLIB_NO_AUTO_LINK
             "_SILENCE_ALL_MS_EXT_DEPRECATION_WARNINGS;"
 		)
 	endif()
+	add_compile_definitions(STORMLIB_NO_AUTO_LINK)
+	set(STORMLIB_NO_AUTO_LINK ON)
 elseif (CMAKE_SYSTEM_NAME STREQUAL "CafeOS")
 	target_compile_definitions(${PROJECT_NAME} PRIVATE
 		"$<$<CONFIG:Debug>:"
@@ -321,16 +327,55 @@ endif()
 
 target_link_libraries(${PROJECT_NAME} PRIVATE "${ADDITIONAL_LIBRARY_DEPENDENCIES}")
 
+include(ExternalProject)
+ExternalProject_Add(TorchExternal
+	PREFIX TorchExternal
+    SOURCE_DIR ${CMAKE_SOURCE_DIR}/tools/Torch
+    CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${CMAKE_BINARY_DIR}/TorchExternal
+)
+ExternalProject_Get_Property(TorchExternal install_dir)
+set(TORCH_EXECUTABLE ${install_dir}/src/TorchExternal-build/$<CONFIGURATION>/torch)
+add_custom_target(
+    ExtractAssetHeaders
+    DEPENDS TorchExternal
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    COMMAND ${TORCH_EXECUTABLE} header -o baserom.us.uncompressed.z64
+)
+add_custom_target(
+    GeneratePortOtr
+    DEPENDS TorchExternal
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    COMMAND ${TORCH_EXECUTABLE} pack port starship.otr
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different "${CMAKE_SOURCE_DIR}/starship.otr" "${CMAKE_BINARY_DIR}/starship.otr"
+)
+add_custom_target(
+    ExtractAssets
+    DEPENDS TorchExternal
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    COMMAND ${TORCH_EXECUTABLE} otr baserom.us.uncompressed.z64
+    COMMAND ${TORCH_EXECUTABLE} pack port starship.otr
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different "${CMAKE_SOURCE_DIR}/sf64.otr" "${CMAKE_BINARY_DIR}/sf64.otr"
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different "${CMAKE_SOURCE_DIR}/starship.otr" "${CMAKE_BINARY_DIR}/starship.otr"
+)
+
+option(USE_STANDALONE "Build as a standalone executable" OFF)
+option(BUILD_STORMLIB "Build with StormLib support" OFF)
+
+option(BUILD_SM64 "Build with Super Mario 64 support" OFF)
+option(BUILD_MK64 "Build with Mario Kart 64 support" OFF)
+add_subdirectory(tools/Torch)
+target_link_libraries(${PROJECT_NAME} PRIVATE torch)
+
 if(CMAKE_SYSTEM_NAME MATCHES "NintendoSwitch")
 
-nx_generate_nacp(Lylat.nacp
+nx_generate_nacp(Starship.nacp
    NAME "${PROJECT_NAME}"
    AUTHOR "${PROJECT_TEAM}"
    VERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH}"
 )
 
 nx_create_nro(${PROJECT_NAME}
-    NACP Lylat.nacp
+    NACP Starship.nacp
 )
 
 INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}.nro DESTINATION . COMPONENT ${PROJECT_NAME})
@@ -526,23 +571,4 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang|AppleClang")
           -Wl,-export-dynamic
       )
   endif()
-endif()
+endif()
-
-include(ExternalProject)
-ExternalProject_Add(Torch
-	PREFIX Torch
-    SOURCE_DIR ${CMAKE_SOURCE_DIR}/tools/Torch
-    CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${CMAKE_BINARY_DIR}/Torch
-)
-ExternalProject_Get_Property(Torch install_dir)
-set(TORCH_EXECUTABLE ${install_dir}/src/Torch-build/$<CONFIGURATION>/torch)
-add_custom_target(
-    ExtractAssets
-    DEPENDS Torch
-    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-    COMMAND ${TORCH_EXECUTABLE} header -o baserom.us.uncompressed.z64
-    COMMAND ${TORCH_EXECUTABLE} otr baserom.us.uncompressed.z64
-    COMMAND ${TORCH_EXECUTABLE} pack port starship.otr
-    COMMAND ${CMAKE_COMMAND} -E copy_if_different "${CMAKE_SOURCE_DIR}/sf64.otr" "${CMAKE_BINARY_DIR}/sf64.otr"
-    COMMAND ${CMAKE_COMMAND} -E copy_if_different "${CMAKE_SOURCE_DIR}/starship.otr" "${CMAKE_BINARY_DIR}/starship.otr"
-)

config.yml

@@ -1,5 +1,14 @@
 # Supported Star Fox 64 Versions: US, JP
 
+09f0d105f476b00efa5303a3ebc42e60a7753b7a:
+  name: Star Fox 64 (U) (V1.1) (Compressed)
+  preprocess:
+    decompress_mio0:
+      method: mio0-comptool
+      type: decompress
+      target: f7475fb11e7e6830f82883412638e8390791ab87
+      restart: true
+
 f7475fb11e7e6830f82883412638e8390791ab87:
   name: Star Fox 64 (U) (V1.1)
   path: assets/yaml/us/rev1
@@ -37,4 +46,16 @@ d064229a32cc05ab85e2381ce07744eb3ffaf530:
       - include/sf64mesg.h
       - include/sf64audio_external.h
       - include/sf64event.h
-      - include/sf64player.h
+      - include/sf64player.h
+
+5f658e88ffa9de23cba6986a8fd3d3a90d7b4340:
+  name: F-ZERO X US REV 0 (v1.0)
+  path: test-yml
+  config:
+    gbi: F3DEX2
+    sort: OFFSET
+    logging: ERROR
+    output:
+      code: src/assets
+      headers: include/assets
+      modding: src/assets

src/port/Engine.cpp

@@ -36,6 +36,7 @@
 #include <VertexFactory.h>
 #include "audio/GameAudio.h"
 #include "port/patches/DisplayListPatch.h"
+#include "port/extractor/GameExtractor.h"
 // #include "sf64audio_provisional.h"
 
 #include <Fast3D/gfx_pc.h>
@@ -61,32 +62,58 @@ void AudioThread_CreateNextAudioBuffer(int16_t* samples, uint32_t num_samples);
 GameEngine* GameEngine::Instance;
 static GamePool MemoryPool = { .chunk = 1024 * 512, .cursor = 0, .length = 0, .memory = nullptr };
 
-GameEngine::GameEngine() {
+void GameEngine::GenAssetFile() {
-    std::vector<std::string> OTRFiles;
+    auto extractor = GameExtractor();
-    if (const std::string cube_path = Ship::Context::GetPathRelativeToAppDirectory("starship.otr");
+
-        std::filesystem::exists(cube_path)) {
+    if (!extractor.SelectGameFromUI()) {
-        OTRFiles.push_back(cube_path);
+        ShowMessage("Extractor", "No game selected.");
+        return;
     }
-    if (const std::string sm64_otr_path = Ship::Context::GetPathRelativeToAppDirectory("sf64.otr");
+    if (!extractor.ValidateChecksum()) {
-        std::filesystem::exists(sm64_otr_path)) {
+        ShowMessage("Extractor", "Invalid checksum.");
-        OTRFiles.push_back(sm64_otr_path);
+        return;
     }
-    if (const std::string patches_path = Ship::Context::GetPathRelativeToAppDirectory("mods");
+
-        !patches_path.empty() && std::filesystem::exists(patches_path)) {
+    extractor.DecompressGame();
+
+    if (!extractor.GenerateOTR()) {
+        ShowMessage("Extractor", "Failed to generate OTR.");
+    }
+}
+
+void GameEngine::Create() {
+    std::vector<std::string> AssetFiles;
+    const auto instance = Instance = new GameEngine();
+    const std::string main_otr_path = Ship::Context::GetPathRelativeToAppDirectory("sf64.otr");
+    const std::string assets_path = Ship::Context::GetPathRelativeToAppDirectory("starship.otr");
+    const std::string patches_path = Ship::Context::GetPathRelativeToAppDirectory("mods");
+
+    if (std::filesystem::exists(main_otr_path)) {
+        AssetFiles.push_back(main_otr_path);
+    } else {
+        GenAssetFile();
+        AssetFiles.push_back(main_otr_path);
+    }
+
+    if (std::filesystem::exists(assets_path)) {
+        AssetFiles.push_back(assets_path);
+    }
+
+    if (!patches_path.empty() && std::filesystem::exists(patches_path)) {
         if (std::filesystem::is_directory(patches_path)) {
             for (const auto& p : std::filesystem::recursive_directory_iterator(patches_path)) {
                 const auto ext = p.path().extension().string();
                 if (StringHelper::IEquals(ext, ".otr") || StringHelper::IEquals(ext, ".o2r")) {
-                    OTRFiles.push_back(p.path().generic_string());
+                    AssetFiles.push_back(p.path().generic_string());
                 }
             }
         }
     }
 
-    this->context =
+    instance->context =
-        Ship::Context::CreateInstance("Starship", "ship", "starship.cfg.json", OTRFiles, {}, 3, { 44100, 1024*2, 2480*2 });
+        Ship::Context::CreateInstance("Starship", "ship", "starship.cfg.json", AssetFiles, {}, 3, { 44100, 1024*2, 2480*2 });
 
-    auto loader = context->GetResourceManager()->GetResourceLoader();
+    auto loader = instance->context->GetResourceManager()->GetResourceLoader();
     loader->RegisterResourceFactory(std::make_shared<SF64::ResourceFactoryBinaryAnimV0>(), RESOURCE_FORMAT_BINARY,
                                     "Animation", static_cast<uint32_t>(SF64::ResourceType::AnimData), 0);
     loader->RegisterResourceFactory(std::make_shared<SF64::ResourceFactoryBinarySkeletonV0>(), RESOURCE_FORMAT_BINARY,
@@ -164,11 +191,8 @@ GameEngine::GameEngine() {
 
     loader->RegisterResourceFactory(std::make_shared<SF64::ResourceFactoryBinarySoundFontV0>(), RESOURCE_FORMAT_BINARY,
                                     "SoundFont", static_cast<uint32_t>(SF64::ResourceType::SoundFont), 0);
-}
 
-void GameEngine::Create() {
+    GameEngine::AudioInit();
-    const auto instance = Instance = new GameEngine();
-    instance->AudioInit();
     DisplayListPatch::Run();
     GameUI::SetupGuiElements();
 #if defined(__SWITCH__) || defined(__WIIU__)
@@ -386,6 +410,15 @@ uint32_t GameEngine::GetInterpolationFPS() {
                               CVarGetInteger("gInterpolationFPS", 20));
 }
 
+void GameEngine::ShowMessage(const char* title, const char* message) {
+#if defined(__SWITCH__)
+    SPDLOG_ERROR(message);
+#else
+    SDL_ShowSimpleMessageBox(SDL_MESSAGEBOX_ERROR, title, message, nullptr);
+    SPDLOG_ERROR(message);
+#endif
+}
+
 extern "C" uint32_t GameEngine_GetSampleRate() {
     auto player = Ship::Context::GetInstance()->GetAudio()->GetAudioPlayer();
     if (player == nullptr) {

src/port/Engine.h

@@ -15,28 +15,26 @@ struct GamePool {
 #include <Fast3D/gfx_pc.h>
 #include "libultraship/src/Context.h"
 
-
-
 class GameEngine {
   public:
     static GameEngine* Instance;
 
     std::shared_ptr<Ship::Context> context;
 
-    GameEngine();
+    GameEngine(){};
+    static void GenAssetFile();
     static void Create();
     void StartFrame() const;
     static void HandleAudioThread();
-    static void ProcessAudioTask(s16* audio_buffer);
     static void StartAudioFrame();
     static void EndAudioFrame();
     static void AudioInit();
     static void AudioExit();
     static void RunCommands(Gfx* Commands, const std::vector<std::unordered_map<Mtx*, MtxF>>& mtx_replacements);
-    void ProcessFrame(void (*run_one_game_iter)()) const;
     static void Destroy();
     static void ProcessGfxCommands(Gfx* commands);
     static uint32_t GetInterpolationFPS();
+    static void ShowMessage(const char* title, const char* message);
 };
 
 extern "C" void* GameEngine_Malloc(size_t size);

src/port/extractor/CompTool.cpp

@@ -0,0 +1,86 @@
+#include "CompTool.h"
+#include "utils/Decompressor.h"
+#include "BinaryReader.h"
+#include "BinaryWriter.h"
+#include <fstream>
+
+uint32_t CompTool::FindFileTable(std::vector<uint8_t>& rom) {
+    uint8_t query_one[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x50, 0x00, 0x00, 0x00, 0x00 };
+    uint8_t query_two[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x60, 0x00, 0x00, 0x00, 0x00 };
+
+    for(size_t i = 0; i < rom.size() - sizeof(query_one); i++) {
+        if(memcmp(rom.data() + i, query_one, sizeof(query_one)) == 0){
+            return i;
+        }
+
+        if(memcmp(rom.data() + i, query_two, sizeof(query_two)) == 0){
+            return i;
+        }
+    }
+
+    throw std::runtime_error("Failed to find file table");
+}
+
+std::vector<uint8_t> CompTool::Decompress(std::vector<uint8_t> rom){
+    Ship::BinaryReader basefile((char*) rom.data(), rom.size());
+    basefile.SetEndianness(Ship::Endianness::Big);
+
+    Ship::BinaryWriter decompfile;
+    decompfile.SetEndianness(Ship::Endianness::Big);
+    decompfile.Write((uint8_t)0x80);
+
+    uint32_t table = CompTool::FindFileTable(rom);
+    uint32_t count = 0;
+    while (true){
+        auto entry = table + 0x10 * count;
+        basefile.Seek(entry, Ship::SeekOffsetType::Start);
+
+        auto v_begin = basefile.ReadInt32();
+        auto p_begin = basefile.ReadInt32();
+        auto p_end = basefile.ReadInt32();
+        auto comp_flag = basefile.ReadInt32();
+
+        auto p_size = p_end - p_begin;
+        auto v_size = (int32_t) 0;
+        DataChunk* decoded = nullptr;
+
+        if(v_begin == 0 && p_end == 0){
+            break;
+        }
+
+        basefile.Seek(p_begin, Ship::SeekOffsetType::Start);
+
+        auto bytes = new uint8_t[p_size];
+        basefile.Read((char*) bytes, p_size);
+
+        switch ((CompType) comp_flag) {
+            case CompType::UNCOMPRESSED:
+                v_size = p_size;
+                break;
+            case CompType::COMPRESSED:
+                decoded = Decompressor::Decode(std::vector(bytes, bytes + p_size), 0, CompressionType::MIO0, true);
+                bytes = decoded->data;
+                v_size = decoded->size;
+                break;
+            default:
+                throw std::runtime_error("Invalid compression flag. There may be a problem with your ROM.");
+        }
+
+        decompfile.Seek(v_begin, Ship::SeekOffsetType::Start);
+        decompfile.Write((char*) bytes, v_size);
+        auto v_end = v_begin + v_size;
+
+        decompfile.Seek(entry + 4, Ship::SeekOffsetType::Start);
+        decompfile.Write(v_begin);
+        decompfile.Write(v_end);
+        decompfile.Write((uint32_t) CompType::UNCOMPRESSED);
+        count++;
+    }
+
+    decompfile.Seek(0x10, Ship::SeekOffsetType::Start);
+    decompfile.Write(0xA7D5F194); // CRC1
+    decompfile.Write(0xFE3DF761); // CRC2
+
+    auto result = decompfile.ToVector();
+    return { (uint8_t*) result.data(), (uint8_t*) result.data() + result.size() };
+}

src/port/extractor/CompTool.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include <cstdint>
+#include <string>
+#include <vector>
+
+enum class CompType {
+    UNCOMPRESSED,
+    COMPRESSED,
+    UNKNOWN
+};
+
+class CompTool {
+public:
+    static std::vector<uint8_t> Decompress(std::vector<uint8_t> rom);
+private:
+    static uint32_t FindFileTable(std::vector<uint8_t>& rom);
+};

src/port/extractor/GameExtractor.cpp

@@ -0,0 +1,63 @@
+#include "GameExtractor.h"
+
+#include <fstream>
+
+#include "Context.h"
+#include "CompTool.h"
+#include "spdlog/spdlog.h"
+#include "portable-file-dialogs.h"
+#include <port/Engine.h>
+
+std::unordered_map<std::string, GameEntry> mGameList = {
+    { "f7475fb11e7e6830f82883412638e8390791ab87", { "Star Fox 64 (U) (V1.1)", false } },
+    { "09f0d105f476b00efa5303a3ebc42e60a7753b7a", { "Star Fox 64 (U) (V1.1)", true } }
+};
+
+bool GameExtractor::SelectGameFromUI() {
+#if !defined(__IOS__) || !defined(__ANDROID__) || !defined(__SWITCH__)
+    auto selection = pfd::open_file("Select a file", ".", { "N64 Roms", "*.z64" }).result();
+
+    if (selection.empty()) {
+        return false;
+    }
+
+    this->mGamePath = selection[0];
+#else
+    this->mGamePath = Ship::Context::GetPathRelativeToAppDirectory("baserom.us.rev1.z64");
+#endif
+
+    std::ifstream file(this->mGamePath, std::ios::binary);
+    this->mGameData = std::vector<uint8_t>( std::istreambuf_iterator( file ), {} );
+    file.close();
+    return true;
+}
+
+std::optional<GameEntry> GameExtractor::ValidateChecksum() const {
+    const auto rom = new N64::Cartridge(this->mGameData);
+    rom->Initialize();
+    return mGameList[rom->GetHash()];
+}
+
+void GameExtractor::DecompressGame() {
+    auto game = this->ValidateChecksum();
+    if (!game.has_value()) {
+        return;
+    }
+
+    if (game->isCompressed) {
+        this->mGameData = CompTool::Decompress(this->mGameData);
+    }
+}
+
+bool GameExtractor::GenerateOTR() const {
+    Companion::Instance = new Companion(this->mGameData, true, false);
+
+    try {
+        Companion::Instance->Init(ExportType::Binary);
+    } catch (const std::exception& e) {
+        GameEngine::ShowMessage("Failed to generate OTR", e.what());
+        exit(1);
+    }
+
+    return true;
+}

src/port/extractor/GameExtractor.h

@@ -0,0 +1,22 @@
+#pragma once
+
+#include "Companion.h"
+#include <filesystem>
+#include <vector>
+#include <cstdint>
+
+struct GameEntry {
+    std::string name;
+    bool isCompressed;
+};
+
+class GameExtractor {
+  public:
+    std::optional<GameEntry> ValidateChecksum() const;
+    bool SelectGameFromUI();
+    void DecompressGame();
+    bool GenerateOTR() const;
+  private:
+    fs::path mGamePath;
+    std::vector<uint8_t> mGameData;
+};