Add UTF-8 filename support on Windows

Author: aqnuep

CMakeLists.txt

@@ -241,6 +241,9 @@ if(MSVC)
     # because `/W4;/WX` is returned as a single string.
     add_compile_options( /W4;$<$<BOOL:${KTX_WERROR}>:/WX> )
     add_compile_options( $<IF:$<CONFIG:Debug>,/Gz,/O2> )
+    # Enable UTF-8 support
+    add_compile_options( $<$<C_COMPILER_ID:MSVC>:/utf-8> )
+    add_compile_options( $<$<CXX_COMPILER_ID:MSVC>:/utf-8> )
 elseif(${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU"
        OR ${CMAKE_CXX_COMPILER_ID} MATCHES "Clang")
     add_compile_options( -Wall -Wextra $<$<BOOL:${KTX_WERROR}>:-Werror>)

tests/cts

@@ -13,8 +13,20 @@
 #include <fmt/ostream.h>
 #include <fmt/printf.h>
 
-#if defined(_WIN32) && defined(DEBUG)
-#include <windows.h> // For functions used by launchDebugger 
+#if defined(_WIN32)
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+
+std::wstring ConvertToUTF16(std::string str) {
+    std::wstring result;
+    int len = MultiByteToWideChar(CP_UTF8, 0, str.c_str(), static_cast<int>(str.length()), NULL, 0);
+    if (len > 0)
+    {
+        result.resize(len);
+        MultiByteToWideChar(CP_UTF8, 0, str.c_str(), static_cast<int>(str.length()), &result[0], len);
+    }
+    return result;
+}
 #endif
 
 // -------------------------------------------------------------------------------------------------
@@ -111,7 +123,11 @@ InputStream::InputStream(const std::string& filepath, Reporter& report) :
         stdinBuffer << std::cin.rdbuf();
         activeStream = &stdinBuffer;
     } else {
+#if defined(_WIN32)
+        file.open(ConvertToUTF16(filepath).c_str(), std::ios::binary | std::ios::in);
+#else
         file.open(filepath, std::ios::binary | std::ios::in);
+#endif
         if (!file)
             report.fatal(rc::IO_FAILURE, "Could not open input file \"{}\": {}.", filepath, errnoMessage());
         activeStream = &file;
@@ -134,7 +150,11 @@ OutputStream::OutputStream(const std::string& filepath, Reporter& report) :
         #endif
         file = stdout;
     } else {
+#if defined(_WIN32)
+        file = _wfopen(ConvertToUTF16(filepath).c_str(), L"wb");
+#else
         file = std::fopen(filepath.c_str(), "wb");
+#endif
         if (!file)
             report.fatal(rc::IO_FAILURE, "Could not open output file \"{}\": {}.", filepath, errnoMessage());
     }

tools/ktx/command.cpp

@@ -7,12 +7,19 @@
 #include "stdafx.h"
 #include <iostream>
 #include <string>
+#include <memory>
 #include <unordered_map>
 
 #include <cxxopts.hpp>
 #include <fmt/ostream.h>
 #include <fmt/printf.h>
 
+#if defined(_WIN32)
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+#include <shellapi.h>
+#endif
+
 // -------------------------------------------------------------------------------------------------
 
 namespace ktx {
@@ -185,6 +192,24 @@ int _tmain(int argc, _TCHAR* argv[]) {
     //      // pbxproj file, so it can't be disabled in a generated project.
     //      // Remove these from the arguments under consideration.
 
+#if defined(_WIN32)
+    // Windows does not support UTF-8 argv so we have to manually acquire it
+    LPWSTR commandLine = GetCommandLineW();
+    LPWSTR* wideArgv = CommandLineToArgvW(commandLine, &argc);
+    std::vector<std::unique_ptr<_TCHAR[]>> utf8Argv(argc);
+    for (int i = 0; i < argc; ++i) {
+        int byteSize = WideCharToMultiByte(CP_UTF8, 0, wideArgv[i], -1, nullptr, 0, nullptr, nullptr);
+        utf8Argv[i] = std::make_unique<_TCHAR[]>(byteSize);
+        WideCharToMultiByte(CP_UTF8, 0, wideArgv[i], -1, utf8Argv[i].get(), byteSize, nullptr, nullptr);
+        argv[i] = utf8Argv[i].get();
+    }
+
+    // Set UTF-8 codepage for the console
+    if (!SetConsoleOutputCP(CP_UTF8)) {
+        fmt::print(std::cerr, "{} warning: failed to set UTF-8 code page for console output.\n", argv[0]);
+    }
+#endif
+
     if (argc >= 2) {
         // Has a subcommand, attempt to lookup