From d4ee0ceb6dec6f2bb9ea47eee5b53bdeb12897bd Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Wed, 19 May 2021 16:24:08 +0100 Subject: [PATCH 01/65] Add astc cli options This is the start of astc support. First cli options are provided. Since ktx-software now supports multiple encoder this patch extracts out the common options into '--encode ' and removes redundant options. Also adds enums for astc compressor into ktx.h. --- include/ktx.h | 133 +++++++++++++++++++++++++++- tools/toktx/toktx.cc | 8 +- utils/scapp.h | 202 ++++++++++++++++++++++++++++++++++--------- 3 files changed, 296 insertions(+), 47 deletions(-) diff --git a/include/ktx.h b/include/ktx.h index 950bb94ebe..4e17e0a016 100644 --- a/include/ktx.h +++ b/include/ktx.h @@ -949,8 +949,140 @@ typedef enum ktx_pack_uastc_flag_bits_e { } ktx_pack_uastc_flag_bits_e; typedef ktx_uint32_t ktx_pack_uastc_flags; +/** + * @~English + * @brief Options specifiying ASTC encoding quality levels. + * Asct encoder provides similar options but I would + * rather keep ktx.h cleaner from other encoder enums. + */ +typedef enum ktx_pack_astc_quality_levels_e { + KTX_PACK_ASTC_QUALITY_LEVEL_FASTEST = 0, + /*!< Fastest compression. */ + KTX_PACK_ASTC_QUALITY_LEVEL_FAST = 10, + /*!< Fast compression. */ + KTX_PACK_ASTC_QUALITY_LEVEL_MEDIUM = 60, + /*!< Medium compression. */ + KTX_PACK_ASTC_QUALITY_LEVEL_THOROUGH = 98, + /*!< Slower compression. */ + KTX_PACK_ASTC_QUALITY_LEVEL_EXHAUSTIVE = 100, + /*!< Very slow compression. */ + KTX_PACK_ASTC_QUALITY_LEVEL_MAX = KTX_PACK_ASTC_QUALITY_LEVEL_EXHAUSTIVE, + /*!< Maximum supported quality level. */ +} ktx_pack_astc_quality_levels_e; + +/** + * @~English + * @brief Options specifiying ASTC encoding block dimensions + */ +typedef enum ktx_pack_astc_block_size_e { + // 2D formats + KTX_PACK_ASTC_BLOCK_4x4, //: 8.00 bpp + KTX_PACK_ASTC_BLOCK_5x4, //: 6.40 bpp + KTX_PACK_ASTC_BLOCK_5x5, //: 5.12 bpp + KTX_PACK_ASTC_BLOCK_6x5, //: 4.27 bpp + KTX_PACK_ASTC_BLOCK_6x6, //: 3.56 bpp + KTX_PACK_ASTC_BLOCK_8x5, //: 3.20 bpp + KTX_PACK_ASTC_BLOCK_8x6, //: 2.67 bpp + KTX_PACK_ASTC_BLOCK_10x5, //: 2.56 bpp + KTX_PACK_ASTC_BLOCK_10x6, //: 2.13 bpp + KTX_PACK_ASTC_BLOCK_8x8, //: 2.00 bpp + KTX_PACK_ASTC_BLOCK_10x8, //: 1.60 bpp + KTX_PACK_ASTC_BLOCK_10x10, //: 1.28 bpp + KTX_PACK_ASTC_BLOCK_12x10, //: 1.07 bpp + KTX_PACK_ASTC_BLOCK_12x12, //: 0.89 bpp + // 3D formats + KTX_PACK_ASTC_BLOCK_3x3x3, //: 4.74 bpp + KTX_PACK_ASTC_BLOCK_4x3x3, //: 3.56 bpp + KTX_PACK_ASTC_BLOCK_4x4x3, //: 2.67 bpp + KTX_PACK_ASTC_BLOCK_4x4x4, //: 2.00 bpp + KTX_PACK_ASTC_BLOCK_5x4x4, //: 1.60 bpp + KTX_PACK_ASTC_BLOCK_5x5x4, //: 1.28 bpp + KTX_PACK_ASTC_BLOCK_5x5x5, //: 1.02 bpp + KTX_PACK_ASTC_BLOCK_6x5x5, //: 0.85 bpp + KTX_PACK_ASTC_BLOCK_6x6x5, //: 0.71 bpp + KTX_PACK_ASTC_BLOCK_6x6x6, //: 0.59 bpp + KTX_PACK_ASTC_BLOACK_SIZE_MAX = KTX_PACK_ASTC_BLOCK_6x6x6 + /*!< Maximum supported blocks. */ +} ktx_pack_astc_block_size_e; + +/** + * @~English + * @brief Options specifiying ASTC encoder profile + */ +typedef enum ktx_pack_astc_encoder_profile_e { + KTX_PACK_ASTC_ENCODER_PROFILE_SRGB_LDR, + KTX_PACK_ASTC_ENCODER_PROFILE_LINEAR_LDR, + KTX_PACK_ASTC_ENCODER_PROFILE_HDR_RGB_LDR_ALPHA, + KTX_PACK_ASTC_ENCODER_PROFILE_HDR_RGBA, + KTX_PACK_ASTC_ENCODER_PROFILE_MAX = KTX_PACK_ASTC_ENCODER_PROFILE_HDR_RGBA +} ktx_pack_astc_encoder_profile_e; + +typedef enum ktx_pack_astc_encoder_function_e { + KTX_PACK_ASTC_ENCODER_FUNCTION_SRGB, + KTX_PACK_ASTC_ENCODER_FUNCTION_LINEAR, + KTX_PACK_ASTC_ENCODER_FUNCTION_MAX = KTX_PACK_ASTC_ENCODER_FUNCTION_LINEAR +} ktx_pack_astc_encoder_function_e; + +typedef enum ktx_pack_astc_encoder_mode_e { + KTX_PACK_ASTC_ENCODER_MODE_LDR, + KTX_PACK_ASTC_ENCODER_MODE_HDR, + KTX_PACK_ASTC_ENCODER_MODE_MAX = KTX_PACK_ASTC_ENCODER_MODE_HDR +} ktx_pack_astc_encoder_mode_e; + extern KTX_API const ktx_uint32_t KTX_ETC1S_DEFAULT_COMPRESSION_LEVEL; +/** + * @memberof ktxTexture + * @~English + * @brief Structure for passing extended parameters to + * ktxTexture_CompressAstc. + * + * Passing a struct initialized to 0 (e.g. " = {};") will use the default + * values. Only those settings to be modified need be non-zero. + */ +typedef struct ktxAstcParams { + ktx_uint32_t structSize; + /*!< Size of this struct. Used so library can tell which version + of struct is being passed. + */ + ktx_bool_t verbose; + /*!< If true, prints Astc encoder operation details to + @c stdout. Not recommended for GUI apps. + */ + ktx_uint32_t threadCount; + /*!< Number of threads used for compression. Default is 1. */ + + /* astcenc params */ + ktx_uint32_t blockSize; + /*!< Combinations of block dimensions that astcenc supports + i.e. 6x6, 8x8, 6x5 etc*/ + + ktx_uint32_t function; + /*!< Can be {linear/srgb} from astcenc*/ + + ktx_uint32_t mode; + /*!< Can be {ldr/hdr} from astcenc*/ + + ktx_uint32_t qualityLevel; + /*!< astcenc supports -fastest, -fast, -medium, -thorough, -exhaustive*/ + + ktx_bool_t normalMap; + /*!< Tunes codec parameters for better quality on normal maps + In this mode normals are compressed to X,Y components + Discarding Z component, reader will need to generate Z + component in shaders. + */ +} ktxAstcParams; + +ktxAstcParams +ktxTexture_AstcDefaultOptions(); + +KTX_API KTX_error_code KTX_APIENTRY +ktxTexture_CompressAstcEx(ktxTexture* This, ktxAstcParams* params); + +KTX_API KTX_error_code KTX_APIENTRY +ktxTexture_CompressAstc(ktxTexture* This, ktx_uint32_t quality); + /** * @memberof ktxTexture2 * @~English @@ -1505,4 +1637,3 @@ Initial release. */ #endif /* KTX_H_A55A6F00956F42F3A137C11929827FE1 */ - diff --git a/tools/toktx/toktx.cc b/tools/toktx/toktx.cc index f3f50a8a4c..ab4eb7d999 100644 --- a/tools/toktx/toktx.cc +++ b/tools/toktx/toktx.cc @@ -724,7 +724,7 @@ toktxApp::main(int argc, _TCHAR *argv[]) image = Image::CreateFromFile(infile, options.assign_oetf == KHR_DF_TRANSFER_UNSPECIFIED, - options.bcmp || options.bopts.uastc); + options.bcmp || options.bopts.uastc || options.astc); if (i == 0) { // First file. @@ -869,8 +869,8 @@ toktxApp::main(int argc, _TCHAR *argv[]) } if (options.inputSwizzle.size() > 0 - // inputSwizzle is handled during BasisU encoding - && !options.bcmp && !options.bopts.uastc) { + // inputSwizzle is handled during BasisU and astc encoding + && !options.bcmp && !options.bopts.uastc && !options.astc) { image->swizzle(options.inputSwizzle); } @@ -1201,7 +1201,7 @@ toktxApp::main(int argc, _TCHAR *argv[]) // Add Swizzle metadata if (options.swizzle.size()) { swizzle = options.swizzle; - } else if (!options.bcmp && !options.bopts.uastc + } else if (!options.bcmp && !options.bopts.uastc && !options.astc && defaultSwizzle.size()) { swizzle = defaultSwizzle; } diff --git a/utils/scapp.h b/utils/scapp.h index 200d05d86a..0f451126d3 100644 --- a/utils/scapp.h +++ b/utils/scapp.h @@ -38,6 +38,18 @@ struct clampedOption T max; }; +ktx_pack_astc_block_size_e +getAstcBlockSize(const char* block_size); + +ktx_pack_astc_quality_levels_e +getAstcQualityLevel(const char* quality); + +ktx_pack_astc_encoder_function_e +getAstcEncoderFunction(const char* function); + +ktx_pack_astc_encoder_mode_e +getAstcEncoderMode(const char* mode); + /* // Markdown doesn't work in files included by snipped{doc} or include{doc} // so the table below has to be laboriously done in html. @@ -246,16 +258,46 @@ class scApp : public ktxApp { } }; + struct astcOptions : public ktxAstcParams { + clampedOption threadCount; + clampedOption blockSize; + clampedOption function; + clampedOption mode; + clampedOption qualityLevel; + + astcOptions() : + threadCount(ktxAstcParams::threadCount, 1, 10000), + blockSize(ktxAstcParams::blockSize, 0, KTX_PACK_ASTC_BLOACK_SIZE_MAX), + function(ktxAstcParams::function, 0, KTX_PACK_ASTC_ENCODER_FUNCTION_MAX), + mode(ktxAstcParams::mode, 0, KTX_PACK_ASTC_ENCODER_MODE_MAX), + qualityLevel(ktxAstcParams::qualityLevel, 0, KTX_PACK_ASTC_QUALITY_LEVEL_MAX) + { + uint32_t tc = thread::hardware_concurrency(); + if (tc == 0) tc = 1; + threadCount.max = tc; + threadCount = tc; + + structSize = sizeof(ktxAstcParams); + blockSize.clear(); + function.clear(); + mode.clear(); + qualityLevel.clear(); + normalMap = false; + } + }; int ktx2; int bcmp; int zcmp; + int astc; clamped zcmpLevel; struct basisOptions bopts; + struct astcOptions astcopts; commandOptions() : zcmpLevel(ZSTD_CLEVEL_DEFAULT, 1U, 22U) { ktx2 = false; bcmp = false; zcmp = false; + astc = false; } }; @@ -276,10 +318,74 @@ class scApp : public ktxApp { return scparams; } + void setEncoder(string encoding) + { + if (encoding == "astc") + options.astc = 1; + else if (encoding == "bcmp") + options.bcmp = 1; + else if (encoding == "uastc") + options.bopts.uastc = 1; + } + void usage() { cerr << - " --bcmp Supercompress the image data with ETC1S / BasisLZ. Implies --t2.\n" + " --encode \n" + " Compress the image data with ETC1S / BasisLZ, high-quality transcodable\n" + " UASTC format or ASTC format. Implies --t2 for all encoding.\n" + " Accepable options are 'bcmp', 'uastc' and 'astc'. With each encoding\n" + " option the following encoding specific options become valid, otherwise.\n" + " they are ignored.\n\n" + " astc:\n" + " Create a texture in high-quality ASTC format.\n" + " --astc_blk_s /\n" + " Specify which block size to use for compressing the textures.\n" + " i.e. --astc_blk_s 6x5 for 2D or --astc_blk_s 6x6x6 for 3D.\n" + " 6x6 is default for 2D.\n\n" + " Supported 2D block sizes are:\n\n" + " 4x4: 8.00 bpp 10x5: 2.56 bpp\n" + " 5x4: 6.40 bpp 10x6: 2.13 bpp\n" + " 5x5: 5.12 bpp 8x8: 2.00 bpp\n" + " 6x5: 4.27 bpp 10x8: 1.60 bpp\n" + " 6x6: 3.56 bpp 10x10: 1.28 bpp\n" + " 8x5: 3.20 bpp 12x10: 1.07 bpp\n" + " 8x6: 2.67 bpp 12x12: 0.89 bpp\n\n" + " Supported 3D block sizes are:\n\n" + " 3x3x3: 4.74 bpp 5x5x4: 1.28 bpp\n" + " 4x3x3: 3.56 bpp 5x5x5: 1.02 bpp\n" + " 4x4x3: 2.67 bpp 6x5x5: 0.85 bpp\n" + " 4x4x4: 2.00 bpp 6x6x5: 0.71 bpp\n" + " 5x4x4: 1.60 bpp 6x6x6: 0.59 bpp\n" + " --astc_func \n" + " Specify which transfer function to use while compressing.\n" + " Use this only if you don't trust the color space info in source image.\n" + " Default is srgb for sRGB color space compression.\n" + " --astc_mode \n" + " Specify which encoding mode to use. LDR is the default.\n" + " --astc_quality \n" + " The quality level configures the quality-performance tradeoff for\n" + " the compressor; more complete searches of the search space improve\n" + " image quality at the expense of compression time. Default is 'medium'\n" + " The quality level can be set to fastest (0) and thorough (100) via the \n" + " following fixed quality presets:\n\n" + " fastest (equivalent to quality = 0)\n" + " fast (equivalent to quality = 10)\n" + " medium (equivalent to quality = 60)\n" + " thorough (equivalent to quality = 98)\n" + " exhaustive (equivalent to quality = 100)\n" + " --astc_normal\n" + " The input texture is a three component linear LDR normal map\n" + " storing unit length normals as (R=X, G=Y, B=Z). The output will\n" + " be a two component X+Y normal map stored as (RGB=X, A=Y),\n" + " optimized for angular error instead of simple PSNR. The Z\n" + " component can be recovered programmatically in shader code by\n" + " using the equation:\n\n" + " nml.xy = texture(...).ga; // Load in [0,1]\n" + " nml.xy = nml.xy * 2.0 - 1.0; // Unpack to [-1,1]\n" + " nml.z = sqrt(1 - dot(nml.xy, nml.xy)); // Compute Z\n" + " bcmp:\n" + " Supercompress the image data with ETC1S / BasisLZ.\n" " RED images will become RGB with RED in each component. RG images\n" " will have R in the RGB part and G in the alpha part of the\n" " compressed texture. When set, the following BasisLZ-related\n" @@ -287,11 +393,6 @@ class scApp : public ktxApp { " --no_multithreading\n" " Disable multithreading. Deprecated. For backward compatibility.\n" " Use --threads 1 instead.\n" - " --threads \n" - " Explicitly set the number of threads to use during compression.\n" - " By default, ETC1S / BasisLZ compression will use the number of\n" - " threads reported by thread::hardware_concurrency or 1 if value\n" - " returned is 0.\n" " --clevel \n" " ETC1S / BasisLZ compression level, an encoding speed vs. quality\n" " tradeoff. Range is [0,5], default is 1. Higher values are slower,\n" @@ -342,10 +443,11 @@ class scApp : public ktxApp { " Disable selector rate distortion optimizations. Slightly faster,\n" " less noisy output, but lower quality per output bit. Default is\n" " to do selector RDO.\n\n" - " --uastc []\n" + " uastc:\n" " Create a texture in high-quality transcodable UASTC format.\n" - " Implies --t2. The optional parameter selects a speed\n" - " vs quality tradeoff as shown in the following table:\n" + " --uastc_level \n" + " This optional parameter selects a speed vs quality\n" + " tradeoff as shown in the following table:\n" "\n" " Level | Speed | Quality\n" " ----- | ------- | -------\n" @@ -359,8 +461,8 @@ class scApp : public ktxApp { " compress the UASTC data. This and any LZ-style compression can\n" " be made more effective by conditioning the UASTC texture data\n" " using the Rate Distortion Optimization (RDO) post-process stage.\n" - " When --uastc is set the following options become available for\n" - " controlling RDO:\n\n" + " When uastc encoding is set the following options become available\n" + " for controlling RDO:\n\n" " --uastc_rdo_l []\n" " Enable UASTC RDO post-processing and optionally set UASTC RDO\n" " quality scalar (lambda) to @e lambda. Lower values yield higher\n" @@ -389,15 +491,21 @@ class scApp : public ktxApp { " deterministic).\n\n" " --no_sse Forbid use of the SSE instruction set. Ignored if CPU does not\n" " support SSE. Only the Basis Universal compressor uses SSE.\n" - " --verbose Print encoder/compressor activity status to stdout. Currently\n" - " only the Basis Universal compressor emits status.\n" " --zcmp []\n" " Supercompress the data with Zstandard. Implies --t2. Can be used\n" - " with data in any format except ETC1S / BasisLZ (--bcmp). Most\n" + " with data in any format except ETC1S / BasisLZ (bcmp). Most\n" " effective with RDO-conditioned UASTC or uncompressed formats. The\n" " optional compressionLevel range is 1 - 22 and the default is 3.\n" " Lower values=faster but give less compression. Values above 20\n" - " should be used with caution as they require more memory.\n"; + " should be used with caution as they require more memory.\n" + " --threads \n" + " Explicitly set the number of threads to use during compression.\n" + " By default, ETC1S / BasisLZ and astc compression will use the number of\n" + " threads reported by thread::hardware_concurrency or 1 if value\n" + " returned is 0.\n" + " --verbose Print encoder/compressor activity status to stdout. Currently\n" + " only both astcencoder and Basis Universal compressor emits status.\n" + "\n"; ktxApp::usage(); cerr << endl << "In case of ambiguity, such as when the last option is one with an optional\n" @@ -414,7 +522,7 @@ scApp::scApp(string& version, string& defaultVersion, : ktxApp(version, defaultVersion, options), options(options) { argparser::option my_option_list[] = { - { "bcmp", argparser::option::no_argument, NULL, 'b' }, + { "encode", argparser::option::required_argument, NULL, 'b' }, { "zcmp", argparser::option::optional_argument, NULL, 'z' }, { "no_multithreading", argparser::option::no_argument, NULL, 'N' }, { "threads", argparser::option::required_argument, NULL, 't' }, @@ -429,14 +537,19 @@ scApp::scApp(string& version, string& defaultVersion, { "no_endpoint_rdo", argparser::option::no_argument, NULL, 1001 }, { "no_selector_rdo", argparser::option::no_argument, NULL, 1002 }, { "no_sse", argparser::option::no_argument, NULL, 1011 }, - { "uastc", argparser::option::optional_argument, NULL, 1003 }, + { "uastc_level", argparser::option::required_argument, NULL, 1003 }, { "uastc_rdo_l", argparser::option::optional_argument, NULL, 1004 }, { "uastc_rdo_d", argparser::option::required_argument, NULL, 1005 }, { "uastc_rdo_b", argparser::option::optional_argument, NULL, 1006 }, { "uastc_rdo_s", argparser::option::optional_argument, NULL, 1007 }, { "uastc_rdo_f", argparser::option::no_argument, NULL, 1008 }, { "uastc_rdo_m", argparser::option::no_argument, NULL, 1009 }, - { "verbose", argparser::option::no_argument, NULL, 1010 } + { "verbose", argparser::option::no_argument, NULL, 1010 }, + { "astc_blk_s", argparser::option::required_argument, NULL, 1012 }, + { "astc_func", argparser::option::required_argument, NULL, 1013 }, + { "astc_mode", argparser::option::required_argument, NULL, 1014 }, + { "astc_quality", argparser::option::required_argument, NULL, 1015 }, + { "astc_normal", argparser::option::no_argument, NULL, 1016 } }; const int lastOptionIndex = sizeof(my_option_list) / sizeof(argparser::option); @@ -483,24 +596,12 @@ scApp::processOption(argparser& parser, int opt) switch (opt) { case 'b': - if (options.zcmp) { - cerr << "Only one of --bcmp and --zcmp can be specified." - << endl; - usage(); - exit(1); - } - if (options.bopts.uastc) { - cerr << "Only one of --bcmp and --uastc can be specified." - << endl; - usage(); - exit(1); - } - options.bcmp = 1; + setEncoder(parser.optarg); options.ktx2 = 1; break; case 'z': if (options.bcmp) { - cerr << "Only one of --bcmp and --zcmp can be specified." + cerr << "Only one of '--encode bcmp' and --zcmp can be specified." << endl; usage(); exit(1); @@ -554,18 +655,11 @@ scApp::processOption(argparser& parser, int opt) break; case 't': options.bopts.threadCount = strtoi(parser.optarg.c_str()); + options.astcopts.threadCount = strtoi(parser.optarg.c_str()); capture = false; break; case 1003: - if (options.bcmp) { - cerr << "Only one of --bcmp and --uastc can be specified." - << endl; - usage(); - exit(1); - } - options.bopts.uastc = 1; - options.ktx2 = 1; - if (parser.optarg.size() > 0) { + { ktx_uint32_t level = strtoi(parser.optarg.c_str()); level = clamp(level, 0, KTX_PACK_UASTC_MAX_LEVEL); // Ensure the last one wins in case of multiple of these args. @@ -604,12 +698,37 @@ scApp::processOption(argparser& parser, int opt) break; case 1010: options.bopts.verbose = true; + options.astcopts.verbose = true; capture = false; break; case 1011: options.bopts.noSSE = true; capture = true; break; + case 1012: // astc_blk_s + options.astcopts.blockSize = getAstcBlockSize(parser.optarg.c_str()); + hasArg = true; + capture = false; + break; + case 1013: // astc_func + options.astcopts.function = getAstcEncoderFunction(parser.optarg.c_str()); + hasArg = true; + capture = false; + break; + case 1014: // astc_mode + options.astcopts.mode = getAstcEncoderMode(parser.optarg.c_str()); + hasArg = true; + capture = false; + break; + case 1015: // astc_quality + options.astcopts.qualityLevel = getAstcQualityLevel(parser.optarg.c_str()); + hasArg = true; + capture = false; + break; + case 1016: // astc_normal + options.astcopts.normalMap = true; + capture = false; + break; default: return false; } @@ -622,4 +741,3 @@ scApp::processOption(argparser& parser, int opt) return true; } - From 8781b6067f5a8a7ea076d5aca0a2aa61dafeb1cc Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Thu, 20 May 2021 02:03:00 +0100 Subject: [PATCH 02/65] Add swizzle option, remove redundant astc enum from ktx.h Add default options for transfer function and fix typos. --- include/ktx.h | 21 +++++++++++---------- utils/scapp.h | 12 +++++------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/include/ktx.h b/include/ktx.h index 4e17e0a016..ebe36f4d77 100644 --- a/include/ktx.h +++ b/include/ktx.h @@ -1001,28 +1001,26 @@ typedef enum ktx_pack_astc_block_size_e { KTX_PACK_ASTC_BLOCK_6x5x5, //: 0.85 bpp KTX_PACK_ASTC_BLOCK_6x6x5, //: 0.71 bpp KTX_PACK_ASTC_BLOCK_6x6x6, //: 0.59 bpp - KTX_PACK_ASTC_BLOACK_SIZE_MAX = KTX_PACK_ASTC_BLOCK_6x6x6 + KTX_PACK_ASTC_BLOCK_SIZE_MAX = KTX_PACK_ASTC_BLOCK_6x6x6 /*!< Maximum supported blocks. */ } ktx_pack_astc_block_size_e; /** * @~English - * @brief Options specifiying ASTC encoder profile + * @brief Options specifiying ASTC encoder profile function */ -typedef enum ktx_pack_astc_encoder_profile_e { - KTX_PACK_ASTC_ENCODER_PROFILE_SRGB_LDR, - KTX_PACK_ASTC_ENCODER_PROFILE_LINEAR_LDR, - KTX_PACK_ASTC_ENCODER_PROFILE_HDR_RGB_LDR_ALPHA, - KTX_PACK_ASTC_ENCODER_PROFILE_HDR_RGBA, - KTX_PACK_ASTC_ENCODER_PROFILE_MAX = KTX_PACK_ASTC_ENCODER_PROFILE_HDR_RGBA -} ktx_pack_astc_encoder_profile_e; - typedef enum ktx_pack_astc_encoder_function_e { + KTX_PACK_ASTC_ENCODER_FUNCTION_UNKNOWN, KTX_PACK_ASTC_ENCODER_FUNCTION_SRGB, KTX_PACK_ASTC_ENCODER_FUNCTION_LINEAR, KTX_PACK_ASTC_ENCODER_FUNCTION_MAX = KTX_PACK_ASTC_ENCODER_FUNCTION_LINEAR } ktx_pack_astc_encoder_function_e; +/** + * @~English + * @brief Options specifiying ASTC encoder profile mode + * This and function is used later to derive the profile. + */ typedef enum ktx_pack_astc_encoder_mode_e { KTX_PACK_ASTC_ENCODER_MODE_LDR, KTX_PACK_ASTC_ENCODER_MODE_HDR, @@ -1072,6 +1070,9 @@ typedef struct ktxAstcParams { Discarding Z component, reader will need to generate Z component in shaders. */ + char inputSwizzle[4]; + /*!< A swizzle to provide as input to astcenc. It must match the regular + expression /^[rgba01]{4}$/.*/ } ktxAstcParams; ktxAstcParams diff --git a/utils/scapp.h b/utils/scapp.h index 0f451126d3..3944995ea2 100644 --- a/utils/scapp.h +++ b/utils/scapp.h @@ -267,7 +267,7 @@ class scApp : public ktxApp { astcOptions() : threadCount(ktxAstcParams::threadCount, 1, 10000), - blockSize(ktxAstcParams::blockSize, 0, KTX_PACK_ASTC_BLOACK_SIZE_MAX), + blockSize(ktxAstcParams::blockSize, 0, KTX_PACK_ASTC_BLOCK_SIZE_MAX), function(ktxAstcParams::function, 0, KTX_PACK_ASTC_ENCODER_FUNCTION_MAX), mode(ktxAstcParams::mode, 0, KTX_PACK_ASTC_ENCODER_MODE_MAX), qualityLevel(ktxAstcParams::qualityLevel, 0, KTX_PACK_ASTC_QUALITY_LEVEL_MAX) @@ -279,7 +279,10 @@ class scApp : public ktxApp { structSize = sizeof(ktxAstcParams); blockSize.clear(); + blockSize = KTX_PACK_ASTC_BLOCK_6x6; function.clear(); + // Default to unknown to have a chance to use to use color space from file + function = KTX_PACK_ASTC_ENCODER_FUNCTION_UNKNOWN; mode.clear(); qualityLevel.clear(); normalMap = false; @@ -360,7 +363,7 @@ class scApp : public ktxApp { " --astc_func \n" " Specify which transfer function to use while compressing.\n" " Use this only if you don't trust the color space info in source image.\n" - " Default is srgb for sRGB color space compression.\n" + " Default is what's read from the image.\n" " --astc_mode \n" " Specify which encoding mode to use. LDR is the default.\n" " --astc_quality \n" @@ -708,26 +711,21 @@ scApp::processOption(argparser& parser, int opt) case 1012: // astc_blk_s options.astcopts.blockSize = getAstcBlockSize(parser.optarg.c_str()); hasArg = true; - capture = false; break; case 1013: // astc_func options.astcopts.function = getAstcEncoderFunction(parser.optarg.c_str()); hasArg = true; - capture = false; break; case 1014: // astc_mode options.astcopts.mode = getAstcEncoderMode(parser.optarg.c_str()); hasArg = true; - capture = false; break; case 1015: // astc_quality options.astcopts.qualityLevel = getAstcQualityLevel(parser.optarg.c_str()); hasArg = true; - capture = false; break; case 1016: // astc_normal options.astcopts.normalMap = true; - capture = false; break; default: return false; From 3f248a29cf1cb129da3562e65be590af8e8b9670 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Fri, 21 May 2021 14:06:50 +0100 Subject: [PATCH 03/65] Add astc compression support Includes build system changes, command line arguments plumbing support Whitespace changes 'tabs->spaces' astc compression support for 2D images including 'layers, mip-levels, cubefaces' Multi-threaded compresssion support using pthreads and pthreads style api on windows --- CMakeLists.txt | 23 ++ include/ktx.h | 170 ++++---- lib/astc_encode.cpp | 963 +++++++++++++++++++++++++++++++++++++++++++ tests/toktx-tests | 21 +- tools/toktx/toktx.cc | 39 +- utils/scapp.h | 311 +++++++------- 6 files changed, 1270 insertions(+), 257 deletions(-) create mode 100644 lib/astc_encode.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index ceb9d6c037..540e97a98a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -442,6 +442,7 @@ target_sources( ktx PRIVATE lib/basis_encode.cpp + lib/astc_encode.cpp ${BASISU_ENCODER_C_SRC} ${BASISU_ENCODER_CXX_SRC} lib/writer1.c @@ -628,6 +629,28 @@ PUBLIC add_subdirectory(interface/basisu_c_binding) +# Add astcenc subdirectory +# Only one architecture is supported at once, if default (avx2) is not available, +# user must define ISA_SSE41, ISA_SSE2 or ISA_NEON themselves + +set(ISA_AVX2 ON) # AVX2 is the default +set(ASTC_LIB_NAME astcenc-avx2-static) + +if (${ISA_SSE41}) + set(ASTC_LIB_NAME astcenc-sse4.1-static) +endif() + +if (${ISA_SSE2}) + set(ASTC_LIB_NAME astcenc-sse2-static) +endif() + +if (${ISA_NEON}) + set(ASTC_LIB_NAME astcenc-neon-static) +endif() + +add_subdirectory(lib/astc-encoder) +target_link_libraries(ktx PRIVATE ${ASTC_LIB_NAME}) + # Tools if(KTX_FEATURE_TOOLS) add_subdirectory(tools) diff --git a/include/ktx.h b/include/ktx.h index ebe36f4d77..d1439e12f9 100644 --- a/include/ktx.h +++ b/include/ktx.h @@ -649,8 +649,9 @@ typedef enum ktxSupercmpScheme { KTX_SS_NONE = 0, /*!< No supercompression. */ KTX_SS_BASIS_LZ = 1, /*!< Basis LZ supercompression. */ KTX_SS_ZSTD = 2, /*!< ZStd supercompression. */ + KTX_SS_ASTC = 4, /*!< ASTC compression. */ KTX_SS_BEGIN_RANGE = KTX_SS_NONE, - KTX_SS_END_RANGE = KTX_SS_ZSTD, + KTX_SS_END_RANGE = KTX_SS_ASTC, KTX_SS_BEGIN_VENDOR_RANGE = 0x10000, KTX_SS_END_VENDOR_RANGE = 0x1ffff, KTX_SS_BEGIN_RESERVED = 0x20000, @@ -952,22 +953,20 @@ typedef ktx_uint32_t ktx_pack_uastc_flags; /** * @~English * @brief Options specifiying ASTC encoding quality levels. - * Asct encoder provides similar options but I would - * rather keep ktx.h cleaner from other encoder enums. */ typedef enum ktx_pack_astc_quality_levels_e { - KTX_PACK_ASTC_QUALITY_LEVEL_FASTEST = 0, - /*!< Fastest compression. */ - KTX_PACK_ASTC_QUALITY_LEVEL_FAST = 10, - /*!< Fast compression. */ - KTX_PACK_ASTC_QUALITY_LEVEL_MEDIUM = 60, - /*!< Medium compression. */ - KTX_PACK_ASTC_QUALITY_LEVEL_THOROUGH = 98, - /*!< Slower compression. */ - KTX_PACK_ASTC_QUALITY_LEVEL_EXHAUSTIVE = 100, - /*!< Very slow compression. */ - KTX_PACK_ASTC_QUALITY_LEVEL_MAX = KTX_PACK_ASTC_QUALITY_LEVEL_EXHAUSTIVE, - /*!< Maximum supported quality level. */ + KTX_PACK_ASTC_QUALITY_LEVEL_FASTEST = 0, + /*!< Fastest compression. */ + KTX_PACK_ASTC_QUALITY_LEVEL_FAST = 10, + /*!< Fast compression. */ + KTX_PACK_ASTC_QUALITY_LEVEL_MEDIUM = 60, + /*!< Medium compression. */ + KTX_PACK_ASTC_QUALITY_LEVEL_THOROUGH = 98, + /*!< Slower compression. */ + KTX_PACK_ASTC_QUALITY_LEVEL_EXHAUSTIVE = 100, + /*!< Very slow compression. */ + KTX_PACK_ASTC_QUALITY_LEVEL_MAX = KTX_PACK_ASTC_QUALITY_LEVEL_EXHAUSTIVE, + /*!< Maximum supported quality level. */ } ktx_pack_astc_quality_levels_e; /** @@ -975,34 +974,34 @@ typedef enum ktx_pack_astc_quality_levels_e { * @brief Options specifiying ASTC encoding block dimensions */ typedef enum ktx_pack_astc_block_size_e { - // 2D formats - KTX_PACK_ASTC_BLOCK_4x4, //: 8.00 bpp - KTX_PACK_ASTC_BLOCK_5x4, //: 6.40 bpp - KTX_PACK_ASTC_BLOCK_5x5, //: 5.12 bpp - KTX_PACK_ASTC_BLOCK_6x5, //: 4.27 bpp - KTX_PACK_ASTC_BLOCK_6x6, //: 3.56 bpp - KTX_PACK_ASTC_BLOCK_8x5, //: 3.20 bpp - KTX_PACK_ASTC_BLOCK_8x6, //: 2.67 bpp - KTX_PACK_ASTC_BLOCK_10x5, //: 2.56 bpp - KTX_PACK_ASTC_BLOCK_10x6, //: 2.13 bpp - KTX_PACK_ASTC_BLOCK_8x8, //: 2.00 bpp - KTX_PACK_ASTC_BLOCK_10x8, //: 1.60 bpp - KTX_PACK_ASTC_BLOCK_10x10, //: 1.28 bpp - KTX_PACK_ASTC_BLOCK_12x10, //: 1.07 bpp - KTX_PACK_ASTC_BLOCK_12x12, //: 0.89 bpp - // 3D formats - KTX_PACK_ASTC_BLOCK_3x3x3, //: 4.74 bpp - KTX_PACK_ASTC_BLOCK_4x3x3, //: 3.56 bpp - KTX_PACK_ASTC_BLOCK_4x4x3, //: 2.67 bpp - KTX_PACK_ASTC_BLOCK_4x4x4, //: 2.00 bpp - KTX_PACK_ASTC_BLOCK_5x4x4, //: 1.60 bpp - KTX_PACK_ASTC_BLOCK_5x5x4, //: 1.28 bpp - KTX_PACK_ASTC_BLOCK_5x5x5, //: 1.02 bpp - KTX_PACK_ASTC_BLOCK_6x5x5, //: 0.85 bpp - KTX_PACK_ASTC_BLOCK_6x6x5, //: 0.71 bpp - KTX_PACK_ASTC_BLOCK_6x6x6, //: 0.59 bpp - KTX_PACK_ASTC_BLOCK_SIZE_MAX = KTX_PACK_ASTC_BLOCK_6x6x6 - /*!< Maximum supported blocks. */ + // 2D formats + KTX_PACK_ASTC_BLOCK_4x4, //: 8.00 bpp + KTX_PACK_ASTC_BLOCK_5x4, //: 6.40 bpp + KTX_PACK_ASTC_BLOCK_5x5, //: 5.12 bpp + KTX_PACK_ASTC_BLOCK_6x5, //: 4.27 bpp + KTX_PACK_ASTC_BLOCK_6x6, //: 3.56 bpp + KTX_PACK_ASTC_BLOCK_8x5, //: 3.20 bpp + KTX_PACK_ASTC_BLOCK_8x6, //: 2.67 bpp + KTX_PACK_ASTC_BLOCK_10x5, //: 2.56 bpp + KTX_PACK_ASTC_BLOCK_10x6, //: 2.13 bpp + KTX_PACK_ASTC_BLOCK_8x8, //: 2.00 bpp + KTX_PACK_ASTC_BLOCK_10x8, //: 1.60 bpp + KTX_PACK_ASTC_BLOCK_10x10, //: 1.28 bpp + KTX_PACK_ASTC_BLOCK_12x10, //: 1.07 bpp + KTX_PACK_ASTC_BLOCK_12x12, //: 0.89 bpp + // 3D formats + KTX_PACK_ASTC_BLOCK_3x3x3, //: 4.74 bpp + KTX_PACK_ASTC_BLOCK_4x3x3, //: 3.56 bpp + KTX_PACK_ASTC_BLOCK_4x4x3, //: 2.67 bpp + KTX_PACK_ASTC_BLOCK_4x4x4, //: 2.00 bpp + KTX_PACK_ASTC_BLOCK_5x4x4, //: 1.60 bpp + KTX_PACK_ASTC_BLOCK_5x5x4, //: 1.28 bpp + KTX_PACK_ASTC_BLOCK_5x5x5, //: 1.02 bpp + KTX_PACK_ASTC_BLOCK_6x5x5, //: 0.85 bpp + KTX_PACK_ASTC_BLOCK_6x6x5, //: 0.71 bpp + KTX_PACK_ASTC_BLOCK_6x6x6, //: 0.59 bpp + KTX_PACK_ASTC_BLOCK_SIZE_MAX = KTX_PACK_ASTC_BLOCK_6x6x6 + /*!< Maximum supported blocks. */ } ktx_pack_astc_block_size_e; /** @@ -1010,10 +1009,10 @@ typedef enum ktx_pack_astc_block_size_e { * @brief Options specifiying ASTC encoder profile function */ typedef enum ktx_pack_astc_encoder_function_e { - KTX_PACK_ASTC_ENCODER_FUNCTION_UNKNOWN, - KTX_PACK_ASTC_ENCODER_FUNCTION_SRGB, - KTX_PACK_ASTC_ENCODER_FUNCTION_LINEAR, - KTX_PACK_ASTC_ENCODER_FUNCTION_MAX = KTX_PACK_ASTC_ENCODER_FUNCTION_LINEAR + KTX_PACK_ASTC_ENCODER_FUNCTION_UNKNOWN, + KTX_PACK_ASTC_ENCODER_FUNCTION_SRGB, + KTX_PACK_ASTC_ENCODER_FUNCTION_LINEAR, + KTX_PACK_ASTC_ENCODER_FUNCTION_MAX = KTX_PACK_ASTC_ENCODER_FUNCTION_LINEAR } ktx_pack_astc_encoder_function_e; /** @@ -1022,9 +1021,9 @@ typedef enum ktx_pack_astc_encoder_function_e { * This and function is used later to derive the profile. */ typedef enum ktx_pack_astc_encoder_mode_e { - KTX_PACK_ASTC_ENCODER_MODE_LDR, - KTX_PACK_ASTC_ENCODER_MODE_HDR, - KTX_PACK_ASTC_ENCODER_MODE_MAX = KTX_PACK_ASTC_ENCODER_MODE_HDR + KTX_PACK_ASTC_ENCODER_MODE_LDR, + KTX_PACK_ASTC_ENCODER_MODE_HDR, + KTX_PACK_ASTC_ENCODER_MODE_MAX = KTX_PACK_ASTC_ENCODER_MODE_HDR } ktx_pack_astc_encoder_mode_e; extern KTX_API const ktx_uint32_t KTX_ETC1S_DEFAULT_COMPRESSION_LEVEL; @@ -1039,44 +1038,41 @@ extern KTX_API const ktx_uint32_t KTX_ETC1S_DEFAULT_COMPRESSION_LEVEL; * values. Only those settings to be modified need be non-zero. */ typedef struct ktxAstcParams { - ktx_uint32_t structSize; - /*!< Size of this struct. Used so library can tell which version - of struct is being passed. - */ - ktx_bool_t verbose; - /*!< If true, prints Astc encoder operation details to - @c stdout. Not recommended for GUI apps. - */ - ktx_uint32_t threadCount; - /*!< Number of threads used for compression. Default is 1. */ - - /* astcenc params */ - ktx_uint32_t blockSize; - /*!< Combinations of block dimensions that astcenc supports - i.e. 6x6, 8x8, 6x5 etc*/ - - ktx_uint32_t function; - /*!< Can be {linear/srgb} from astcenc*/ - - ktx_uint32_t mode; - /*!< Can be {ldr/hdr} from astcenc*/ - - ktx_uint32_t qualityLevel; - /*!< astcenc supports -fastest, -fast, -medium, -thorough, -exhaustive*/ - - ktx_bool_t normalMap; - /*!< Tunes codec parameters for better quality on normal maps - In this mode normals are compressed to X,Y components - Discarding Z component, reader will need to generate Z - component in shaders. - */ - char inputSwizzle[4]; - /*!< A swizzle to provide as input to astcenc. It must match the regular - expression /^[rgba01]{4}$/.*/ -} ktxAstcParams; + ktx_uint32_t structSize; + /*!< Size of this struct. Used so library can tell which version + of struct is being passed. + */ + ktx_bool_t verbose; + /*!< If true, prints Astc encoder operation details to + @c stdout. Not recommended for GUI apps. + */ + ktx_uint32_t threadCount; + /*!< Number of threads used for compression. Default is 1. */ + + /* astcenc params */ + ktx_uint32_t blockSize; + /*!< Combinations of block dimensions that astcenc supports + i.e. 6x6, 8x8, 6x5 etc*/ + + ktx_uint32_t function; + /*!< Can be {linear/srgb} from astcenc*/ + + ktx_uint32_t mode; + /*!< Can be {ldr/hdr} from astcenc*/ + + ktx_uint32_t qualityLevel; + /*!< astcenc supports -fastest, -fast, -medium, -thorough, -exhaustive*/ -ktxAstcParams -ktxTexture_AstcDefaultOptions(); + ktx_bool_t normalMap; + /*!< Tunes codec parameters for better quality on normal maps + In this mode normals are compressed to X,Y components + Discarding Z component, reader will need to generate Z + component in shaders. + */ + char inputSwizzle[4]; + /*!< A swizzle to provide as input to astcenc. It must match the regular + expression /^[rgba01]{4}$/.*/ +} ktxAstcParams; KTX_API KTX_error_code KTX_APIENTRY ktxTexture_CompressAstcEx(ktxTexture* This, ktxAstcParams* params); diff --git a/lib/astc_encode.cpp b/lib/astc_encode.cpp new file mode 100644 index 0000000000..31305e2fc1 --- /dev/null +++ b/lib/astc_encode.cpp @@ -0,0 +1,963 @@ +/* -*- tab-width: 4; -*- */ +/* vi: set sw=2 ts=4 expandtab: */ + +/* + * Copyright (c) 2021, Arm Limited and Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * @internal + * @file astc_encode.cpp + * @~English + * + * @brief Functions for compressing a texture to astc format. + * + * @author Wasim Abbas , www.arm.com + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "dfd.h" +#include "ktx.h" +#include "ktxint.h" +#include "texture2.h" +#include "vkformat_enum.h" +#include "vk_format.h" + +#include "astcenc.h" +#include "../tools/toktx/image.hpp" + +// Provide pthreads support on windows +#if defined(_WIN32) && !defined(__CYGWIN__) + +#define WIN32_LEAN_AND_MEAN +#include + +typedef HANDLE pthread_t; +typedef int pthread_attr_t; + +/* Public function, see header file for detailed documentation */ +static int +pthread_create(pthread_t* thread, const pthread_attr_t* attribs, + void* (*threadfunc)(void*), void* thread_arg) { + (void)attribs; + LPTHREAD_START_ROUTINE func = (LPTHREAD_START_ROUTINE)threadfunc; + *thread = CreateThread(nullptr, 0, func, thread_arg, 0, nullptr); + return 0; +} + +/* Public function, see header file for detailed documentation */ +static int +pthread_join(pthread_t thread, void** value) { + (void)value; + WaitForSingleObject(thread, INFINITE); + return 0; +} + +#endif + +static astcenc_image* +imageAllocate(uint32_t bitness, + uint32_t dim_x, uint32_t dim_y, uint32_t dim_z) { + astcenc_image *img = new astcenc_image; + assert(img); + + img->dim_x = dim_x; + img->dim_y = dim_y; + img->dim_z = dim_z; + + if (bitness == 8) { + void **data = new void *[dim_z]; + img->data_type = ASTCENC_TYPE_U8; + img->data = data; + + for (uint32_t z = 0; z < dim_z; z++) { + data[z] = new uint8_t[dim_x * dim_y * 4]; + } + } + else if (bitness == 16) { + void **data = new void *[dim_z]; + img->data_type = ASTCENC_TYPE_F16; + img->data = data; + + for (uint32_t z = 0; z < dim_z; z++) { + data[z] = new uint16_t[dim_x * dim_y * 4]; + } + } + else { // if (bitness == 32) + assert(bitness == 32); + void **data = new void *[dim_z]; + img->data_type = ASTCENC_TYPE_F32; + img->data = data; + + for (uint32_t z = 0; z < dim_z; z++) { + data[z] = new float[dim_x * dim_y * 4]; + } + } + + return img; +} + +static void +imageFree(astcenc_image *img) { + if (img == nullptr) { + return; + } + + for (uint32_t z = 0; z < img->dim_z; z++) { + delete[](char *) img->data[z]; + } + + delete[] img->data; + delete img; +} + +static astcenc_image* +unorm8x1ArrayToImage(const uint8_t *data, uint32_t dim_x, uint32_t dim_y) { + astcenc_image *img = imageAllocate(8, dim_x, dim_y, 1); + assert(img); + + for (uint32_t y = 0; y < dim_y; y++) { + uint8_t * data8 = static_cast(img->data[0]); + const uint8_t *src = data + dim_x * y; + + for (uint32_t x = 0; x < dim_x; x++) { + data8[(4 * dim_x * y) + (4 * x) ] = src[x]; + data8[(4 * dim_x * y) + (4 * x + 1)] = src[x]; + data8[(4 * dim_x * y) + (4 * x + 2)] = src[x]; + data8[(4 * dim_x * y) + (4 * x + 3)] = 255; + } + } + + return img; +} + +static astcenc_image* +unorm8x2ArrayToImage(const uint8_t *data, uint32_t dim_x, uint32_t dim_y) { + astcenc_image *img = imageAllocate(8, dim_x, dim_y, 1); + assert(img); + + for (uint32_t y = 0; y < dim_y; y++) { + uint8_t * data8 = static_cast(img->data[0]); + const uint8_t *src = data + 2 * dim_x * y; + + for (uint32_t x = 0; x < dim_x; x++) { + data8[(4 * dim_x * y) + (4 * x) ] = src[2 * x ]; + data8[(4 * dim_x * y) + (4 * x + 1)] = src[2 * x ]; + data8[(4 * dim_x * y) + (4 * x + 2)] = src[2 * x ]; + data8[(4 * dim_x * y) + (4 * x + 3)] = src[2 * x + 1]; + } + } + + return img; +} + +static astcenc_image* +unorm8x3ArrayToImage(const uint8_t *data, uint32_t dim_x, uint32_t dim_y) { + astcenc_image *img = imageAllocate(8, dim_x, dim_y, 1); + assert(img); + + for (uint32_t y = 0; y < dim_y; y++) { + uint8_t * data8 = static_cast(img->data[0]); + const uint8_t *src = data + 3 * dim_x * y; + + for (uint32_t x = 0; x < dim_x; x++) { + data8[(4 * dim_x * y) + (4 * x) ] = src[3 * x ]; + data8[(4 * dim_x * y) + (4 * x + 1)] = src[3 * x + 1]; + data8[(4 * dim_x * y) + (4 * x + 2)] = src[3 * x + 2]; + data8[(4 * dim_x * y) + (4 * x + 3)] = 255; + } + } + + return img; +} + +static astcenc_image* +unorm8x4ArrayToImage(const uint8_t *data, uint32_t dim_x, uint32_t dim_y) { + astcenc_image *img = imageAllocate(8, dim_x, dim_y, 1); + assert(img); + + for (uint32_t y = 0; y < dim_y; y++) { + uint8_t * data8 = static_cast(img->data[0]); + const uint8_t *src = data + 4 * dim_x * y; + + for (uint32_t x = 0; x < dim_x; x++) { + data8[(4 * dim_x * y) + (4 * x) ] = src[4 * x ]; + data8[(4 * dim_x * y) + (4 * x + 1)] = src[4 * x + 1]; + data8[(4 * dim_x * y) + (4 * x + 2)] = src[4 * x + 2]; + data8[(4 * dim_x * y) + (4 * x + 3)] = src[4 * x + 3]; + } + } + + return img; +} + +static ktx_size_t +astcBufferSize(uint32_t width, uint32_t height, uint32_t depth, + uint32_t block_x, uint32_t block_y, uint32_t block_z) { + auto xblocs = (width + block_x - 1) / block_x; + auto yblocs = (height + block_y - 1) / block_y; + auto zblocs = (depth + block_z - 1) / block_z; + + return xblocs * yblocs * zblocs * 16; +} + +/** + * @memberof ktxTexture + * @ingroup write + * @~English + * @brief Creates default astc parameters + * + * @return ktxAstcParams with default options for astc compressor + */ +static ktxAstcParams +astcDefaultOptions() { + ktxAstcParams params{}; + params.structSize = sizeof(params); + params.verbose = false; + params.threadCount = 1; + params.blockSize = KTX_PACK_ASTC_BLOCK_6x6; + params.function = KTX_PACK_ASTC_ENCODER_FUNCTION_UNKNOWN; + params.mode = KTX_PACK_ASTC_ENCODER_MODE_LDR; + params.qualityLevel = KTX_PACK_ASTC_QUALITY_LEVEL_MEDIUM; + params.normalMap = false; + + return params; +} + +/** + * @memberof ktxTexture + * @ingroup write + * @~English + * @brief Creates valid astc block size from string. + * + * @return Valid ktx_pack_astc_block_size_e from string + */ +ktx_pack_astc_block_size_e +astcBlockSize(const char* block_size) { + static std::unordered_map + astc_blocks_mapping{{"4x4", KTX_PACK_ASTC_BLOCK_4x4}, + {"5x4", KTX_PACK_ASTC_BLOCK_5x4}, + {"5x5", KTX_PACK_ASTC_BLOCK_5x5}, + {"6x5", KTX_PACK_ASTC_BLOCK_6x5}, + {"6x6", KTX_PACK_ASTC_BLOCK_6x6}, + {"8x5", KTX_PACK_ASTC_BLOCK_8x5}, + {"8x6", KTX_PACK_ASTC_BLOCK_8x6}, + {"10x5", KTX_PACK_ASTC_BLOCK_10x5}, + {"10x6", KTX_PACK_ASTC_BLOCK_10x6}, + {"8x8", KTX_PACK_ASTC_BLOCK_8x8}, + {"10x8", KTX_PACK_ASTC_BLOCK_10x8}, + {"10x10", KTX_PACK_ASTC_BLOCK_10x10}, + {"12x10", KTX_PACK_ASTC_BLOCK_12x10}, + {"12x12", KTX_PACK_ASTC_BLOCK_12x12}, + {"3x3x3", KTX_PACK_ASTC_BLOCK_3x3x3}, + {"4x3x3", KTX_PACK_ASTC_BLOCK_4x3x3}, + {"4x4x3", KTX_PACK_ASTC_BLOCK_4x4x3}, + {"4x4x4", KTX_PACK_ASTC_BLOCK_4x4x4}, + {"5x4x4", KTX_PACK_ASTC_BLOCK_5x4x4}, + {"5x5x4", KTX_PACK_ASTC_BLOCK_5x5x4}, + {"5x5x5", KTX_PACK_ASTC_BLOCK_5x5x5}, + {"6x5x5", KTX_PACK_ASTC_BLOCK_6x5x5}, + {"6x6x5", KTX_PACK_ASTC_BLOCK_6x6x5}, + {"6x6x6", KTX_PACK_ASTC_BLOCK_6x6x6}}; + + auto opt = astc_blocks_mapping.find(block_size); + + if (opt != astc_blocks_mapping.end()) + return opt->second; + + return KTX_PACK_ASTC_BLOCK_6x6; +} + +/** + * @memberof ktxTexture + * @ingroup write + * @~English + * @brief Creates valid astc quality from string. + * + * @return Valid ktx_pack_astc_quality_e from string + */ +ktx_pack_astc_quality_levels_e +astcQualityLevel(const char *quality) { + + static std::unordered_map astc_quality_mapping{ + {"fastest", KTX_PACK_ASTC_QUALITY_LEVEL_FASTEST}, + {"fast", KTX_PACK_ASTC_QUALITY_LEVEL_FAST}, + {"medium", KTX_PACK_ASTC_QUALITY_LEVEL_MEDIUM}, + {"thorough", KTX_PACK_ASTC_QUALITY_LEVEL_THOROUGH}, + {"exhaustive", KTX_PACK_ASTC_QUALITY_LEVEL_EXHAUSTIVE} + }; + + auto opt = astc_quality_mapping.find(quality); + + if (opt != astc_quality_mapping.end()) + return opt->second; + + return KTX_PACK_ASTC_QUALITY_LEVEL_MEDIUM; +} + +/** + * @memberof ktxTexture + * @ingroup write + * @~English + * @brief Creates valid astc function from string. + * Checks the input file color space to use as default. + * + * @return Valid ktx_pack_astc_function_e from string + */ +ktx_pack_astc_encoder_function_e +astcEncoderFunction(const char* function) { + if (std::strcmp(function, "srgb") == 0) + return KTX_PACK_ASTC_ENCODER_FUNCTION_SRGB; + else if (std::strcmp(function, "linear") == 0) + return KTX_PACK_ASTC_ENCODER_FUNCTION_LINEAR; + + return KTX_PACK_ASTC_ENCODER_FUNCTION_SRGB; +} + +/** + * @memberof ktxTexture + * @ingroup write + * @~English + * @brief Creates valid astc mode from string. + * + * @return Valid ktx_pack_astc_mode_e from string + */ +ktx_pack_astc_encoder_mode_e +astcEncoderMode(const char* mode) { + if (std::strcmp(mode, "ldr") == 0) + return KTX_PACK_ASTC_ENCODER_MODE_LDR; + else if (std::strcmp(mode, "hdr") == 0) + return KTX_PACK_ASTC_ENCODER_MODE_HDR; + + return KTX_PACK_ASTC_ENCODER_MODE_LDR; +} + +/** + * @memberof ktxTexture + * @ingroup write + * @~English + * @brief Should be used to get VkFormat from astc block enum + * + * @return VKFormat for a specific astc block size + */ +static ktx_uint32_t +astcVkFormat(ktx_uint32_t block_size, bool sRGB) { + if (sRGB) { + switch (block_size) { + case KTX_PACK_ASTC_BLOCK_4x4: return VK_FORMAT_ASTC_4x4_SRGB_BLOCK; + case KTX_PACK_ASTC_BLOCK_5x4: return VK_FORMAT_ASTC_5x4_SRGB_BLOCK; + case KTX_PACK_ASTC_BLOCK_5x5: return VK_FORMAT_ASTC_5x5_SRGB_BLOCK; + case KTX_PACK_ASTC_BLOCK_6x5: return VK_FORMAT_ASTC_6x5_SRGB_BLOCK; + case KTX_PACK_ASTC_BLOCK_6x6: return VK_FORMAT_ASTC_6x6_SRGB_BLOCK; + case KTX_PACK_ASTC_BLOCK_8x5: return VK_FORMAT_ASTC_8x5_SRGB_BLOCK; + case KTX_PACK_ASTC_BLOCK_8x6: return VK_FORMAT_ASTC_8x6_SRGB_BLOCK; + case KTX_PACK_ASTC_BLOCK_8x8: return VK_FORMAT_ASTC_8x8_SRGB_BLOCK; + case KTX_PACK_ASTC_BLOCK_10x5: return VK_FORMAT_ASTC_10x5_SRGB_BLOCK; + case KTX_PACK_ASTC_BLOCK_10x6: return VK_FORMAT_ASTC_10x6_SRGB_BLOCK; + case KTX_PACK_ASTC_BLOCK_10x8: return VK_FORMAT_ASTC_10x8_SRGB_BLOCK; + case KTX_PACK_ASTC_BLOCK_10x10: return VK_FORMAT_ASTC_10x10_SRGB_BLOCK; + case KTX_PACK_ASTC_BLOCK_12x10: return VK_FORMAT_ASTC_12x10_SRGB_BLOCK; + case KTX_PACK_ASTC_BLOCK_12x12: return VK_FORMAT_ASTC_12x12_SRGB_BLOCK; + case KTX_PACK_ASTC_BLOCK_3x3x3: return VK_FORMAT_ASTC_3x3x3_SRGB_BLOCK_EXT; + case KTX_PACK_ASTC_BLOCK_4x3x3: return VK_FORMAT_ASTC_4x3x3_SRGB_BLOCK_EXT; + case KTX_PACK_ASTC_BLOCK_4x4x3: return VK_FORMAT_ASTC_4x4x3_SRGB_BLOCK_EXT; + case KTX_PACK_ASTC_BLOCK_4x4x4: return VK_FORMAT_ASTC_4x4x4_SRGB_BLOCK_EXT; + case KTX_PACK_ASTC_BLOCK_5x4x4: return VK_FORMAT_ASTC_5x4x4_SRGB_BLOCK_EXT; + case KTX_PACK_ASTC_BLOCK_5x5x4: return VK_FORMAT_ASTC_5x5x4_SRGB_BLOCK_EXT; + case KTX_PACK_ASTC_BLOCK_5x5x5: return VK_FORMAT_ASTC_5x5x5_SRGB_BLOCK_EXT; + case KTX_PACK_ASTC_BLOCK_6x5x5: return VK_FORMAT_ASTC_6x5x5_SRGB_BLOCK_EXT; + case KTX_PACK_ASTC_BLOCK_6x6x5: return VK_FORMAT_ASTC_6x6x5_SRGB_BLOCK_EXT; + case KTX_PACK_ASTC_BLOCK_6x6x6: return VK_FORMAT_ASTC_6x6x6_SRGB_BLOCK_EXT; + } + } else { + switch (block_size) { + case KTX_PACK_ASTC_BLOCK_4x4: return VK_FORMAT_ASTC_4x4_UNORM_BLOCK; + case KTX_PACK_ASTC_BLOCK_5x4: return VK_FORMAT_ASTC_5x4_UNORM_BLOCK; + case KTX_PACK_ASTC_BLOCK_5x5: return VK_FORMAT_ASTC_5x5_UNORM_BLOCK; + case KTX_PACK_ASTC_BLOCK_6x5: return VK_FORMAT_ASTC_6x5_UNORM_BLOCK; + case KTX_PACK_ASTC_BLOCK_6x6: return VK_FORMAT_ASTC_6x6_UNORM_BLOCK; + case KTX_PACK_ASTC_BLOCK_8x5: return VK_FORMAT_ASTC_8x5_UNORM_BLOCK; + case KTX_PACK_ASTC_BLOCK_8x6: return VK_FORMAT_ASTC_8x6_UNORM_BLOCK; + case KTX_PACK_ASTC_BLOCK_8x8: return VK_FORMAT_ASTC_8x8_UNORM_BLOCK; + case KTX_PACK_ASTC_BLOCK_10x5: return VK_FORMAT_ASTC_10x5_UNORM_BLOCK; + case KTX_PACK_ASTC_BLOCK_10x6: return VK_FORMAT_ASTC_10x6_UNORM_BLOCK; + case KTX_PACK_ASTC_BLOCK_10x8: return VK_FORMAT_ASTC_10x8_UNORM_BLOCK; + case KTX_PACK_ASTC_BLOCK_10x10: return VK_FORMAT_ASTC_10x10_UNORM_BLOCK; + case KTX_PACK_ASTC_BLOCK_12x10: return VK_FORMAT_ASTC_12x10_UNORM_BLOCK; + case KTX_PACK_ASTC_BLOCK_12x12: return VK_FORMAT_ASTC_12x12_UNORM_BLOCK; + case KTX_PACK_ASTC_BLOCK_3x3x3: return VK_FORMAT_ASTC_3x3x3_UNORM_BLOCK_EXT; + case KTX_PACK_ASTC_BLOCK_4x3x3: return VK_FORMAT_ASTC_4x3x3_UNORM_BLOCK_EXT; + case KTX_PACK_ASTC_BLOCK_4x4x3: return VK_FORMAT_ASTC_4x4x3_UNORM_BLOCK_EXT; + case KTX_PACK_ASTC_BLOCK_4x4x4: return VK_FORMAT_ASTC_4x4x4_UNORM_BLOCK_EXT; + case KTX_PACK_ASTC_BLOCK_5x4x4: return VK_FORMAT_ASTC_5x4x4_UNORM_BLOCK_EXT; + case KTX_PACK_ASTC_BLOCK_5x5x4: return VK_FORMAT_ASTC_5x5x4_UNORM_BLOCK_EXT; + case KTX_PACK_ASTC_BLOCK_5x5x5: return VK_FORMAT_ASTC_5x5x5_UNORM_BLOCK_EXT; + case KTX_PACK_ASTC_BLOCK_6x5x5: return VK_FORMAT_ASTC_6x5x5_UNORM_BLOCK_EXT; + case KTX_PACK_ASTC_BLOCK_6x6x5: return VK_FORMAT_ASTC_6x6x5_UNORM_BLOCK_EXT; + case KTX_PACK_ASTC_BLOCK_6x6x6: return VK_FORMAT_ASTC_6x6x6_UNORM_BLOCK_EXT; + } + } + + return VK_FORMAT_ASTC_6x6_SRGB_BLOCK; // Default is 6x6 sRGB image +} + +/** + * @memberof ktxTexture + * @ingroup write + * @~English + * @brief Creates valid astc encoder action from string. + * + * @return Valid astc_profile from string + */ +static astcenc_profile +astcEncoderAction(const ktxAstcParams ¶ms, const uint32_t* bdb) { + + if (params.function == KTX_PACK_ASTC_ENCODER_FUNCTION_SRGB && + params.mode == KTX_PACK_ASTC_ENCODER_MODE_LDR) { + return ASTCENC_PRF_LDR_SRGB; + } + else if (params.function == KTX_PACK_ASTC_ENCODER_FUNCTION_LINEAR && + params.mode == KTX_PACK_ASTC_ENCODER_MODE_LDR) { + return ASTCENC_PRF_LDR; + } + else if (params.function == KTX_PACK_ASTC_ENCODER_FUNCTION_LINEAR && + params.mode == KTX_PACK_ASTC_ENCODER_MODE_HDR) { + return ASTCENC_PRF_HDR; + } + else if (params.function == KTX_PACK_ASTC_ENCODER_FUNCTION_UNKNOWN) { + // If no options provided assume the user wants to use + // color space info provided from the file + + ktx_uint32_t transfer = KHR_DFDVAL(bdb, TRANSFER); + if (transfer == KHR_DF_TRANSFER_SRGB && + params.mode == KTX_PACK_ASTC_ENCODER_MODE_LDR) + return ASTCENC_PRF_LDR_SRGB; + else if (transfer == KHR_DF_TRANSFER_LINEAR) { + if (params.mode == KTX_PACK_ASTC_ENCODER_MODE_LDR) + return ASTCENC_PRF_LDR; + else + return ASTCENC_PRF_HDR; + } + } + // TODO: Add support for the following + // KTX_PACK_ASTC_ENCODER_ACTION_COMP_HDR_RGB_LDR_ALPHA; not supported + + return ASTCENC_PRF_LDR_SRGB; +} + + +/** + * @memberof ktxTexture + * @ingroup write + * @~English + * @brief Creates valid astc encoder swizzle from string. + * + * @return Valid astcenc_swizzle from string + */ +static astcenc_swizzle +astcSwizzle(const ktxAstcParams ¶ms) { + + astcenc_swizzle swizzle{ASTCENC_SWZ_R, ASTCENC_SWZ_G, ASTCENC_SWZ_B, ASTCENC_SWZ_A}; + + std::vector swizzle_array{&swizzle.r, &swizzle.g, &swizzle.b, &swizzle.a}; + + for (int i = 0; i < 4; i++) { + if (params.inputSwizzle[i] == 'r') + *swizzle_array[i] = ASTCENC_SWZ_R; + else if (params.inputSwizzle[i] == 'g') + *swizzle_array[i] = ASTCENC_SWZ_G; + else if (params.inputSwizzle[i] == 'b') + *swizzle_array[i] = ASTCENC_SWZ_B; + else if (params.inputSwizzle[i] == 'a') + *swizzle_array[i] = ASTCENC_SWZ_A; + else if (params.inputSwizzle[i] == '0') + *swizzle_array[i] = ASTCENC_SWZ_0; + else if (params.inputSwizzle[i] == '1') + *swizzle_array[i] = ASTCENC_SWZ_1; + } + + return swizzle; +} + +static void +astcBlockSizes(ktx_uint32_t block_size, + uint32_t& block_x, uint32_t& block_y, uint32_t& block_z) { + switch (block_size) { + case KTX_PACK_ASTC_BLOCK_4x4 : block_x = 4; block_y = 4; block_z = 1; break; + case KTX_PACK_ASTC_BLOCK_5x4 : block_x = 5; block_y = 4; block_z = 1; break; + case KTX_PACK_ASTC_BLOCK_5x5 : block_x = 5; block_y = 5; block_z = 1; break; + case KTX_PACK_ASTC_BLOCK_6x5 : block_x = 6; block_y = 5; block_z = 1; break; + case KTX_PACK_ASTC_BLOCK_6x6 : block_x = 6; block_y = 6; block_z = 1; break; + case KTX_PACK_ASTC_BLOCK_8x5 : block_x = 8; block_y = 5; block_z = 1; break; + case KTX_PACK_ASTC_BLOCK_8x6 : block_x = 8; block_y = 6; block_z = 1; break; + case KTX_PACK_ASTC_BLOCK_10x5 : block_x = 10; block_y = 5; block_z = 1; break; + case KTX_PACK_ASTC_BLOCK_10x6 : block_x = 10; block_y = 6; block_z = 1; break; + case KTX_PACK_ASTC_BLOCK_8x8 : block_x = 8; block_y = 8; block_z = 1; break; + case KTX_PACK_ASTC_BLOCK_10x8 : block_x = 10; block_y = 8; block_z = 1; break; + case KTX_PACK_ASTC_BLOCK_10x10 : block_x = 10; block_y = 10; block_z = 1; break; + case KTX_PACK_ASTC_BLOCK_12x10 : block_x = 12; block_y = 10; block_z = 1; break; + case KTX_PACK_ASTC_BLOCK_12x12 : block_x = 12; block_y = 12; block_z = 1; break; + case KTX_PACK_ASTC_BLOCK_3x3x3 : block_x = 3; block_y = 3; block_z = 3; break; + case KTX_PACK_ASTC_BLOCK_4x3x3 : block_x = 4; block_y = 3; block_z = 3; break; + case KTX_PACK_ASTC_BLOCK_4x4x3 : block_x = 4; block_y = 4; block_z = 3; break; + case KTX_PACK_ASTC_BLOCK_4x4x4 : block_x = 4; block_y = 4; block_z = 4; break; + case KTX_PACK_ASTC_BLOCK_5x4x4 : block_x = 5; block_y = 4; block_z = 4; break; + case KTX_PACK_ASTC_BLOCK_5x5x4 : block_x = 5; block_y = 5; block_z = 4; break; + case KTX_PACK_ASTC_BLOCK_5x5x5 : block_x = 5; block_y = 5; block_z = 5; break; + case KTX_PACK_ASTC_BLOCK_6x5x5 : block_x = 6; block_y = 5; block_z = 5; break; + case KTX_PACK_ASTC_BLOCK_6x6x5 : block_x = 6; block_y = 6; block_z = 5; break; + case KTX_PACK_ASTC_BLOCK_6x6x6 : block_x = 6; block_y = 6; block_z = 6; break; + default: + block_x = 6; block_y = 6; block_z = 1; break; + } +} + +static float +astcQuality(ktx_uint32_t quality_level) { + switch (quality_level) { + case KTX_PACK_ASTC_QUALITY_LEVEL_FASTEST: return ASTCENC_PRE_FASTEST; + case KTX_PACK_ASTC_QUALITY_LEVEL_FAST: return ASTCENC_PRE_FAST; + case KTX_PACK_ASTC_QUALITY_LEVEL_MEDIUM: return ASTCENC_PRE_MEDIUM; + case KTX_PACK_ASTC_QUALITY_LEVEL_THOROUGH: return ASTCENC_PRE_THOROUGH; + case KTX_PACK_ASTC_QUALITY_LEVEL_EXHAUSTIVE: return ASTCENC_PRE_EXHAUSTIVE; + } + + return ASTCENC_PRE_MEDIUM; +} + +struct CompressionWorkload { + astcenc_context* context; + astcenc_image* image; + astcenc_swizzle swizzle; + uint8_t* data_out; + size_t data_len; + astcenc_error error; +}; + +static void +compressionWorkloadRunner(int threadCount, int threadId, void* payload) { + (void)threadCount; + + CompressionWorkload* work = static_cast(payload); + astcenc_error error = astcenc_compress_image( + work->context, work->image, &work->swizzle, + work->data_out, work->data_len, threadId); + + // This is a racy update, so which error gets returned is a random, but it + // will reliably report an error if an error occurs + if (error != ASTCENC_SUCCESS) { + work->error = error; + } +} + +/** + * @brief Worker thread helper payload for launchThreads. + */ +struct LaunchDesc { + /** The native thread handle. */ + pthread_t threadHandle; + /** The total number of threads in the thread pool. */ + int threadCount; + /** The thread index in the thread pool. */ + int threadId; + /** The user thread function to execute. */ + void (*func)(int, int, void*); + /** The user thread payload. */ + void* payload; +}; + +/** + * @brief Helper function to translate thread entry points. + * + * Convert a (void*) thread entry to an (int, void*) thread entry, where the + * integer contains the thread ID in the thread pool. + * + * @param p The thread launch helper payload. + */ +static void* +launchThreadsHelper(void *p) { + LaunchDesc* ltd = (LaunchDesc*)p; + ltd->func(ltd->threadCount, ltd->threadId, ltd->payload); + return nullptr; +} + +/* Public function, see header file for detailed documentation */ +static void +launchThreads(int threadCount, void (*func)(int, int, void*), void *payload) { + // Directly execute single threaded workloads on this thread + if (threadCount <= 1) { + func(1, 0, payload); + return; + } + + // Otherwise spawn worker threads + LaunchDesc *threadDescs = new LaunchDesc[threadCount]; + for (int i = 0; i < threadCount; i++) { + threadDescs[i].threadCount = threadCount; + threadDescs[i].threadId = i; + threadDescs[i].payload = payload; + threadDescs[i].func = func; + + pthread_create(&(threadDescs[i].threadHandle), nullptr, + launchThreadsHelper, (void*)&(threadDescs[i])); + } + + // ... and then wait for them to complete + for (int i = 0; i < threadCount; i++) { + pthread_join(threadDescs[i].threadHandle, nullptr); + } + + delete[] threadDescs; +} + +/** + * @memberof ktxTexture + * @ingroup writer + * @~English + * @brief Encode and compress a ktx texture with uncompressed images to astc. + * + * The images are encoded to astc block-compressed format. The encoded images + * replace the original images and the texture's fields including the DFD are + * modified to reflect the new state. + * + * Such textures can be directly uploaded to a GPU via a graphics API. + * + * @param[in] This pointer to the ktxTexture object of interest. + * @param[in] params pointer to astc params object. + * + * @return KTX_SUCCESS on success, other KTX_* enum values on error. + * + * @exception KTX_INVALID_OPERATION + * The texture is already supercompressed. + * @exception KTX_INVALID_OPERATION + * The texture's image are in a block compressed + * format. + * @exception KTX_INVALID_OPERATION + * The texture image's format is a packed format + * (e.g. RGB565). + * @exception KTX_INVALID_OPERATION + * The texture image format's component size is not + 8-bits. + * @exception KTX_INVALID_OPERATION + * The texture's images are 1D. Only 2D images can + * be supercompressed. + * @exception KTX_INVALID_OPERATION + * Astc compressor failed to compress image for any + reason. + * @exception KTX_OUT_OF_MEMORY Not enough memory to carry out supercompression. + */ +extern "C" KTX_error_code +ktxTexture_CompressAstcEx(ktxTexture* _This, ktxAstcParams* params) { + // FIXME: At the moment defaults to ktx2 textures only + assert(_This->classId == ktxTexture2_c && "Only support ktx2 astc."); + ktxTexture2* This = (ktxTexture2*)_This; + + KTX_error_code result; + + if (!params) + return KTX_INVALID_VALUE; + + if (params->structSize != sizeof(struct ktxAstcParams)) + return KTX_INVALID_VALUE; + + if (This->supercompressionScheme != KTX_SS_NONE) + return KTX_INVALID_OPERATION; // Can't apply multiple schemes. + + if (This->isCompressed) + return KTX_INVALID_OPERATION; // Only non-block compressed formats + // can be encoded into a astc format. + + if (This->_protected->_formatSize.flags & KTX_FORMAT_SIZE_PACKED_BIT) + return KTX_INVALID_OPERATION; + + // Basic descriptor block begins after the total size field. + const uint32_t* BDB = This->pDfd+1; + + uint32_t num_components, component_size; + getDFDComponentInfoUnpacked(This->pDfd, &num_components, &component_size); + + if (component_size != 1) + return KTX_INVALID_OPERATION; // Can only deal with 8-bit components + + if (This->pData == NULL) { + result = ktxTexture2_LoadImageData((ktxTexture2*)This, nullptr, 0); + + if (result != KTX_SUCCESS) + return result; + } + + ktx_uint32_t threadCount = params->threadCount; + if (threadCount < 1) + threadCount = 1; + + astcenc_profile profile{ASTCENC_PRF_LDR_SRGB}; + + astcenc_swizzle swizzle{ASTCENC_SWZ_R, ASTCENC_SWZ_G, ASTCENC_SWZ_B, ASTCENC_SWZ_A}; + + uint32_t block_size_x{6}; + uint32_t block_size_y{6}; + uint32_t block_size_z{1}; + float quality{ASTCENC_PRE_MEDIUM}; + uint32_t flags{params->normalMap ? ASTCENC_FLG_MAP_NORMAL : 0}; + + astcBlockSizes(params->blockSize, + block_size_x, block_size_y, block_size_z); + quality = astcQuality(params->qualityLevel); + profile = astcEncoderAction(*params, BDB); + swizzle = astcSwizzle(*params); + + astcenc_config astc_config; + astcenc_context *astc_context; + astcenc_error astc_error = astcenc_config_init(profile, + block_size_x, block_size_y, block_size_z, + quality, flags, + &astc_config); + + if (astc_error != ASTCENC_SUCCESS) { + std::cout << "Astc config init failed\n"; + return KTX_INVALID_OPERATION; + } + + astc_error = astcenc_context_alloc(&astc_config, threadCount, + &astc_context); + + if (astc_error != ASTCENC_SUCCESS) { + std::cout << "Astc context alloc failed\n"; + return KTX_INVALID_OPERATION; + } + + // + // Copy images into compressor readable format. + // + std::vector astc_input_images; + std::vector astc_image_sizes; + + // Conservative allocation, only used to resize data vectors + uint32_t num_images = MAX(1, This->numLayers) * MAX(1, This->numFaces) * + MAX(1, This->numLevels) * MAX(1, This->baseDepth); + + astc_input_images.reserve(num_images); + astc_image_sizes.reserve(num_images); + + // Size of all astc compressed images in bytes + ktx_size_t astc_images_size = 0; + + ktxTexture2_private& priv = *This->_private; + + ktx_size_t level_offset = 0; + + // Walk in reverse on levels so we don't have to do this later + // This->numLevels = 0 not allowed for block compressed formats + for (int32_t level = This->numLevels - 1; level >= 0; level--) { + uint32_t width = MAX(1, This->baseWidth >> level); + uint32_t height = MAX(1, This->baseHeight >> level); + uint32_t depth = MAX(1, This->baseDepth >> level); + uint32_t faceSlices = This->numFaces == 1 ? depth : This->numFaces; + + priv._levelIndex[level].byteLength = 0; + priv._levelIndex[level].uncompressedByteLength = 0; + + ktx_size_t size = astcBufferSize(width, height, depth, + block_size_x, block_size_y, block_size_z); + + for (uint32_t layer = 0; layer < MAX(1, This->numLayers); layer++) { + for (ktx_uint32_t slice = 0; slice < faceSlices; slice++) { + ktx_size_t offset; + ktxTexture2_GetImageOffset((ktxTexture2*)This, level, layer, + slice, &offset); + + // TODO: Fix depth for 3D textures + // Create compressor readable image from each image in the container + astcenc_image *input_image = nullptr; + if (num_components == 1) + input_image = unorm8x1ArrayToImage(This->pData + offset, + width, height); + else if (num_components == 2) + input_image = unorm8x2ArrayToImage(This->pData + offset, + width, height); + else if (num_components == 3) + input_image = unorm8x3ArrayToImage(This->pData + offset, + width, height); + else // assume (num_components == 4) + input_image = unorm8x4ArrayToImage(This->pData + offset, + width, height); + + assert(input_image); + + astc_input_images.push_back(input_image); + astc_image_sizes.push_back(size); + + astc_images_size += size; + + priv._levelIndex[level].byteLength += size; + priv._levelIndex[level].uncompressedByteLength += size; + } + } + + // Offset is from start of the astc compressed block + priv._levelIndex[level].byteOffset = level_offset; + priv._requiredLevelAlignment = 8; // For astc its always 8 bytes (128bits) irrespective of block sizes + + ktx_size_t prev_level_offset = level_offset; + level_offset += _KTX_PADN(priv._requiredLevelAlignment, + priv._levelIndex[level].byteLength); + + ktx_size_t diff = level_offset - prev_level_offset; + + assert(diff - priv._levelIndex[level].byteLength == 0 + && "\nSome format/blocks aren't alighned to 8 bytes, need to create offsets\n"); + } + + free(This->pData); // No longer needed. Reduce memory footprint. + This->pData = NULL; + This->dataSize = 0; + + // Allocate big enough buffer for all compressed images + This->dataSize = astc_images_size; + This->pData = reinterpret_cast(malloc(This->dataSize)); + if (!This->pData) { + return KTX_OUT_OF_MEMORY; + } + + uint8_t* buffer_out = This->pData; + uint32_t input_size = astc_input_images.size(); + + for (uint32_t level = 0; level < input_size; level++) { + if (params->verbose) + std::cout << "Astc compressor: compressing image = " << + level + 1 << " of " << input_size << std::endl; + + // Lets compress to astc + astcenc_image *input_image = astc_input_images[level]; + ktx_size_t size = astc_image_sizes[level]; + + CompressionWorkload work; + work.context = astc_context; + work.image = input_image; + work.swizzle = swizzle; + work.data_out = buffer_out; + work.data_len = size; + work.error = ASTCENC_SUCCESS; + + launchThreads(threadCount, compressionWorkloadRunner, &work); + + if (work.error != ASTCENC_SUCCESS) { + std::cout << "Astc compressor failed\n" << + astcenc_get_error_string(work.error) << std::endl; + + for(auto& ii : astc_input_images) + imageFree(ii); + + astcenc_context_free(astc_context); + return KTX_INVALID_OPERATION; + } + + buffer_out += size; + + // Free input image + imageFree(input_image); + + // Reset astc context for next image + astcenc_compress_reset(astc_context); + } + + // We are done with astcencoder + astcenc_context_free(astc_context); + + ktx_uint32_t transfer = KHR_DFDVAL(BDB, TRANSFER); + bool sRGB = transfer == KHR_DF_TRANSFER_SRGB; // Is this right or I need "KHR_DF_PRIMARIES_SRGB"? + + This->vkFormat = astcVkFormat(params->blockSize, sRGB); + + free(This->pDfd); + This->pDfd = vk2dfd(static_cast(This->vkFormat)); + + // reset pointer to new BDB + BDB = This->pDfd+1; + + // Astc-related checks + ktx_uint32_t model = KHR_DFDVAL(BDB, MODEL); + ktx_uint32_t primaries = KHR_DFDVAL(BDB, PRIMARIES); + This->supercompressionScheme = KTX_SS_ASTC; + This->isCompressed = true; + + assert(model == KHR_DF_MODEL_ASTC && "Invalid dfd generated for astc image\n"); + if (transfer == KHR_DF_TRANSFER_SRGB) { + assert(primaries == KHR_DF_PRIMARIES_SRGB && "Not a valid sRGB image\n"); + } + + // Since we only allow 8-bit components to be compressed + This->_protected->_typeSize = 1; + + return KTX_SUCCESS; +} + +/** + * @memberof ktxTexture + * @ingroup writer + * @~English + * @brief Encode and compress a ktx texture with uncompressed images to astc. + * + * The images are either encoded to astc block-compressed format. The encoded images + * replace the original images and the texture's fields including the DFD are modified to reflect the new + * state. + * + * Such textures can be directly uploaded to a GPU via a graphics API. + * + * @memberof ktxTexture + * @ingroup writer + * @~English + * + * @param[in] This pointer to the ktxTexture object of interest. + * @param[in] quality Compression quality, a value from 0 - 100. + Higher=higher quality/slower speed. + Lower=lower quality/faster speed. + Negative values for quality are considered > 100. + * + * @return KTX_SUCCESS on success, other KTX_* enum values on error. + * + * @exception KTX_INVALID_OPERATION + * The texture is already supercompressed. + * @exception KTX_INVALID_OPERATION + * The texture's image are in a block compressed + * format. + * @exception KTX_INVALID_OPERATION + * The texture image's format is a packed format + * (e.g. RGB565). + * @exception KTX_INVALID_OPERATION + * The texture image format's component size is not 8-bits. + * @exception KTX_INVALID_OPERATION + * The texture's images are 1D. Only 2D images can + * be supercompressed. + * @exception KTX_OUT_OF_MEMORY Not enough memory to carry out supercompression. + */ +extern "C" KTX_error_code +ktxTexture_CompressAstc(ktxTexture* This, ktx_uint32_t quality) { + ktxAstcParams params = astcDefaultOptions(); + + if (quality >= KTX_PACK_ASTC_QUALITY_LEVEL_FASTEST) + params.qualityLevel = KTX_PACK_ASTC_QUALITY_LEVEL_FASTEST; + + if (quality >= KTX_PACK_ASTC_QUALITY_LEVEL_FAST) + params.qualityLevel = KTX_PACK_ASTC_QUALITY_LEVEL_FAST; + + if (quality >= KTX_PACK_ASTC_QUALITY_LEVEL_MEDIUM) + params.qualityLevel = KTX_PACK_ASTC_QUALITY_LEVEL_MEDIUM; + + if (quality >= KTX_PACK_ASTC_QUALITY_LEVEL_THOROUGH) + params.qualityLevel = KTX_PACK_ASTC_QUALITY_LEVEL_THOROUGH; + + if (quality >= KTX_PACK_ASTC_QUALITY_LEVEL_EXHAUSTIVE) + params.qualityLevel = KTX_PACK_ASTC_QUALITY_LEVEL_EXHAUSTIVE; + + return ktxTexture_CompressAstcEx(This, ¶ms); +} diff --git a/tests/toktx-tests b/tests/toktx-tests index d1df637665..b0dbc15fb8 100755 --- a/tests/toktx-tests +++ b/tests/toktx-tests @@ -164,10 +164,19 @@ function gencmpktx() { local args local reference=$1; shift local tempfile=$(mktmp) + local prevarg="" for i in $*; do + if [ ! -z "$prevarg" ]; then + args="$args $i" + prevarg="" + shift + fi if [ ${i:0:2} == "--" ]; then args="$args $i" - shift + if [ ${i:2:6} == "encode" ]; then + prevarg="encode" + fi + shift fi done #echo $toktx $args $tempfile $* @@ -188,11 +197,11 @@ gencmpktx rgb-mipmap-reference.ktx --lower_left_maps_to_s0t0 --mipmap --nometada gencmpktx rgb-mipmap-reference-u.ktx2 --test --t2 --mipmap ../srcimages/level0.ppm ../srcimages/level1.ppm ../srcimages/level2.ppm ../srcimages/level3.ppm ../srcimages/level4.ppm ../srcimages/level5.ppm ../srcimages/level6.ppm if [ $(uname) == "Darwin" ]; then # Run only on macOS until we figure out the Basis compressor non-determinancy. - gencmpktx alpha_simple_basis.ktx2 --test --bcmp ../srcimages/alpha_simple.png - gencmpktx kodim17_basis.ktx2 --test --bcmp ../srcimages/kodim17.png - gencmpktx color_grid_basis.ktx2 --test --bcmp ../srcimages/color_grid.png - gencmpktx cimg5293_uastc.ktx2 --uastc --genmipmap --test ../srcimages/CIMG5293.jpg - gencmpktx cimg5293_uastc_zstd.ktx2 --zcmp --uastc --genmipmap --test ../srcimages/CIMG5293.jpg + gencmpktx alpha_simple_basis.ktx2 --test --encode bcmp ../srcimages/alpha_simple.png + gencmpktx kodim17_basis.ktx2 --test --encode bcmp ../srcimages/kodim17.png + gencmpktx color_grid_basis.ktx2 --test --encode bcmp ../srcimages/color_grid.png + gencmpktx cimg5293_uastc.ktx2 --encode uastc --genmipmap --test ../srcimages/CIMG5293.jpg + gencmpktx cimg5293_uastc_zstd.ktx2 --zcmp --encode uastc --genmipmap --test ../srcimages/CIMG5293.jpg fi filelist=$(mktemp) diff --git a/tools/toktx/toktx.cc b/tools/toktx/toktx.cc index ab4eb7d999..e975c9fee9 100644 --- a/tools/toktx/toktx.cc +++ b/tools/toktx/toktx.cc @@ -724,7 +724,7 @@ toktxApp::main(int argc, _TCHAR *argv[]) image = Image::CreateFromFile(infile, options.assign_oetf == KHR_DF_TRANSFER_UNSPECIFIED, - options.bcmp || options.bopts.uastc || options.astc); + options.bcmp || options.bopts.uastc || options.astc); if (i == 0) { // First file. @@ -869,8 +869,8 @@ toktxApp::main(int argc, _TCHAR *argv[]) } if (options.inputSwizzle.size() > 0 - // inputSwizzle is handled during BasisU and astc encoding - && !options.bcmp && !options.bopts.uastc && !options.astc) { + // inputSwizzle is handled during BasisU and astc encoding + && !options.bcmp && !options.bopts.uastc && !options.astc) { image->swizzle(options.inputSwizzle); } @@ -1201,7 +1201,7 @@ toktxApp::main(int argc, _TCHAR *argv[]) // Add Swizzle metadata if (options.swizzle.size()) { swizzle = options.swizzle; - } else if (!options.bcmp && !options.bopts.uastc && !options.astc + } else if (!options.bcmp && !options.bopts.uastc && !options.astc && defaultSwizzle.size()) { swizzle = defaultSwizzle; } @@ -1257,6 +1257,29 @@ toktxApp::main(int argc, _TCHAR *argv[]) exitCode = 2; goto cleanup; } + } else if (options.astc) { + commandOptions::astcOptions& astcopts = options.astcopts; +#if TRAVIS_DEBUG + astcopts.print(); +#endif + if (options.inputSwizzle.size()) { + for (i = 0; i < 4; i++) { + astcopts.inputSwizzle[i] = options.inputSwizzle[i]; + } + } else if (defaultSwizzle.size()) { + for (i = 0; i < 4; i++) { + astcopts.inputSwizzle[i] = defaultSwizzle[i]; + } + } + + ret = ktxTexture_CompressAstcEx(texture, &astcopts); + if (KTX_SUCCESS != ret) { + fprintf(stderr, "%s failed to compress KTX file \"%s\" to astc; KTX error: %s\n", + name.c_str(), options.outfile.c_str(), + ktxErrorString(ret)); + exitCode = 2; + goto cleanup; + } } else { ret = KTX_SUCCESS; } @@ -1489,7 +1512,7 @@ toktxApp::processOption(argparser& parser, int opt) break; case 1102: std::for_each(parser.optarg.begin(), parser.optarg.end(), [](char & c) { - c = (char)::toupper(c); + c = (char)::toupper(c); }); if (parser.optarg.compare("R") == 0) options.targetType = commandOptions::eR; @@ -1508,7 +1531,7 @@ toktxApp::processOption(argparser& parser, int opt) break; case 1103: std::for_each(parser.optarg.begin(), parser.optarg.end(), [](char & c) { - c = (char)::tolower(c); + c = (char)::tolower(c); }); if (parser.optarg.compare("linear") == 0) options.convert_oetf = KHR_DF_TRANSFER_LINEAR; @@ -1517,7 +1540,7 @@ toktxApp::processOption(argparser& parser, int opt) break; case 1104: std::for_each(parser.optarg.begin(), parser.optarg.end(), [](char & c) { - c = (char)::tolower(c); + c = (char)::tolower(c); }); if (parser.optarg.compare("linear") == 0) options.assign_oetf = KHR_DF_TRANSFER_LINEAR; @@ -1526,7 +1549,7 @@ toktxApp::processOption(argparser& parser, int opt) break; case 1105: std::for_each(parser.optarg.begin(), parser.optarg.end(), [](char & c) { - c = (char)::tolower(c); + c = (char)::tolower(c); }); if (parser.optarg.compare("bt709") == 0) options.assign_primaries = KHR_DF_PRIMARIES_BT709; diff --git a/utils/scapp.h b/utils/scapp.h index 3944995ea2..69c105efa8 100644 --- a/utils/scapp.h +++ b/utils/scapp.h @@ -39,16 +39,16 @@ struct clampedOption }; ktx_pack_astc_block_size_e -getAstcBlockSize(const char* block_size); +astcBlockSize(const char* block_size); ktx_pack_astc_quality_levels_e -getAstcQualityLevel(const char* quality); +astcQualityLevel(const char* quality); ktx_pack_astc_encoder_function_e -getAstcEncoderFunction(const char* function); +astcEncoderFunction(const char* function); ktx_pack_astc_encoder_mode_e -getAstcEncoderMode(const char* mode); +astcEncoderMode(const char* mode); /* // Markdown doesn't work in files included by snipped{doc} or include{doc} @@ -258,49 +258,49 @@ class scApp : public ktxApp { } }; - struct astcOptions : public ktxAstcParams { - clampedOption threadCount; - clampedOption blockSize; - clampedOption function; - clampedOption mode; - clampedOption qualityLevel; - - astcOptions() : - threadCount(ktxAstcParams::threadCount, 1, 10000), - blockSize(ktxAstcParams::blockSize, 0, KTX_PACK_ASTC_BLOCK_SIZE_MAX), - function(ktxAstcParams::function, 0, KTX_PACK_ASTC_ENCODER_FUNCTION_MAX), - mode(ktxAstcParams::mode, 0, KTX_PACK_ASTC_ENCODER_MODE_MAX), - qualityLevel(ktxAstcParams::qualityLevel, 0, KTX_PACK_ASTC_QUALITY_LEVEL_MAX) - { - uint32_t tc = thread::hardware_concurrency(); - if (tc == 0) tc = 1; - threadCount.max = tc; - threadCount = tc; - - structSize = sizeof(ktxAstcParams); - blockSize.clear(); - blockSize = KTX_PACK_ASTC_BLOCK_6x6; - function.clear(); - // Default to unknown to have a chance to use to use color space from file - function = KTX_PACK_ASTC_ENCODER_FUNCTION_UNKNOWN; - mode.clear(); - qualityLevel.clear(); - normalMap = false; - } - }; + struct astcOptions : public ktxAstcParams { + clampedOption threadCount; + clampedOption blockSize; + clampedOption function; + clampedOption mode; + clampedOption qualityLevel; + + astcOptions() : + threadCount(ktxAstcParams::threadCount, 1, 10000), + blockSize(ktxAstcParams::blockSize, 0, KTX_PACK_ASTC_BLOCK_SIZE_MAX), + function(ktxAstcParams::function, 0, KTX_PACK_ASTC_ENCODER_FUNCTION_MAX), + mode(ktxAstcParams::mode, 0, KTX_PACK_ASTC_ENCODER_MODE_MAX), + qualityLevel(ktxAstcParams::qualityLevel, 0, KTX_PACK_ASTC_QUALITY_LEVEL_MAX) + { + uint32_t tc = thread::hardware_concurrency(); + if (tc == 0) tc = 1; + threadCount.max = tc; + threadCount = tc; + + structSize = sizeof(ktxAstcParams); + blockSize.clear(); + blockSize = KTX_PACK_ASTC_BLOCK_6x6; + function.clear(); + // Default to unknown to have a chance to use to use color space from file + function = KTX_PACK_ASTC_ENCODER_FUNCTION_UNKNOWN; + mode.clear(); + qualityLevel.clear(); + normalMap = false; + } + }; int ktx2; int bcmp; int zcmp; - int astc; + int astc; clamped zcmpLevel; struct basisOptions bopts; - struct astcOptions astcopts; + struct astcOptions astcopts; commandOptions() : zcmpLevel(ZSTD_CLEVEL_DEFAULT, 1U, 22U) { ktx2 = false; bcmp = false; zcmp = false; - astc = false; + astc = false; } }; @@ -321,74 +321,73 @@ class scApp : public ktxApp { return scparams; } - void setEncoder(string encoding) - { - if (encoding == "astc") - options.astc = 1; - else if (encoding == "bcmp") - options.bcmp = 1; - else if (encoding == "uastc") - options.bopts.uastc = 1; - } + void setEncoder(string encoding) { + if (encoding == "astc") + options.astc = 1; + else if (encoding == "bcmp") + options.bcmp = 1; + else if (encoding == "uastc") + options.bopts.uastc = 1; + } void usage() { cerr << - " --encode \n" - " Compress the image data with ETC1S / BasisLZ, high-quality transcodable\n" - " UASTC format or ASTC format. Implies --t2 for all encoding.\n" - " Accepable options are 'bcmp', 'uastc' and 'astc'. With each encoding\n" - " option the following encoding specific options become valid, otherwise.\n" - " they are ignored.\n\n" - " astc:\n" - " Create a texture in high-quality ASTC format.\n" - " --astc_blk_s /\n" - " Specify which block size to use for compressing the textures.\n" - " i.e. --astc_blk_s 6x5 for 2D or --astc_blk_s 6x6x6 for 3D.\n" - " 6x6 is default for 2D.\n\n" - " Supported 2D block sizes are:\n\n" - " 4x4: 8.00 bpp 10x5: 2.56 bpp\n" - " 5x4: 6.40 bpp 10x6: 2.13 bpp\n" - " 5x5: 5.12 bpp 8x8: 2.00 bpp\n" - " 6x5: 4.27 bpp 10x8: 1.60 bpp\n" - " 6x6: 3.56 bpp 10x10: 1.28 bpp\n" - " 8x5: 3.20 bpp 12x10: 1.07 bpp\n" - " 8x6: 2.67 bpp 12x12: 0.89 bpp\n\n" - " Supported 3D block sizes are:\n\n" - " 3x3x3: 4.74 bpp 5x5x4: 1.28 bpp\n" - " 4x3x3: 3.56 bpp 5x5x5: 1.02 bpp\n" - " 4x4x3: 2.67 bpp 6x5x5: 0.85 bpp\n" - " 4x4x4: 2.00 bpp 6x6x5: 0.71 bpp\n" - " 5x4x4: 1.60 bpp 6x6x6: 0.59 bpp\n" - " --astc_func \n" - " Specify which transfer function to use while compressing.\n" - " Use this only if you don't trust the color space info in source image.\n" - " Default is what's read from the image.\n" - " --astc_mode \n" - " Specify which encoding mode to use. LDR is the default.\n" - " --astc_quality \n" - " The quality level configures the quality-performance tradeoff for\n" - " the compressor; more complete searches of the search space improve\n" - " image quality at the expense of compression time. Default is 'medium'\n" - " The quality level can be set to fastest (0) and thorough (100) via the \n" - " following fixed quality presets:\n\n" - " fastest (equivalent to quality = 0)\n" - " fast (equivalent to quality = 10)\n" - " medium (equivalent to quality = 60)\n" - " thorough (equivalent to quality = 98)\n" - " exhaustive (equivalent to quality = 100)\n" - " --astc_normal\n" - " The input texture is a three component linear LDR normal map\n" - " storing unit length normals as (R=X, G=Y, B=Z). The output will\n" - " be a two component X+Y normal map stored as (RGB=X, A=Y),\n" - " optimized for angular error instead of simple PSNR. The Z\n" - " component can be recovered programmatically in shader code by\n" - " using the equation:\n\n" - " nml.xy = texture(...).ga; // Load in [0,1]\n" - " nml.xy = nml.xy * 2.0 - 1.0; // Unpack to [-1,1]\n" - " nml.z = sqrt(1 - dot(nml.xy, nml.xy)); // Compute Z\n" - " bcmp:\n" - " Supercompress the image data with ETC1S / BasisLZ.\n" + " --encode \n" + " Compress the image data with ETC1S / BasisLZ, high-quality transcodable\n" + " UASTC format or ASTC format. Implies --t2 for all encoding.\n" + " Accepable options are 'bcmp', 'uastc' and 'astc'. With each encoding\n" + " option the following encoding specific options become valid, otherwise.\n" + " they are ignored.\n\n" + " astc:\n" + " Create a texture in high-quality ASTC format.\n" + " --astc_blk_s /\n" + " Specify which block size to use for compressing the textures.\n" + " e.g. --astc_blk_s 6x5 for 2D or --astc_blk_s 6x6x6 for 3D.\n" + " 6x6 is default for 2D.\n\n" + " Supported 2D block sizes are:\n\n" + " 4x4: 8.00 bpp 10x5: 2.56 bpp\n" + " 5x4: 6.40 bpp 10x6: 2.13 bpp\n" + " 5x5: 5.12 bpp 8x8: 2.00 bpp\n" + " 6x5: 4.27 bpp 10x8: 1.60 bpp\n" + " 6x6: 3.56 bpp 10x10: 1.28 bpp\n" + " 8x5: 3.20 bpp 12x10: 1.07 bpp\n" + " 8x6: 2.67 bpp 12x12: 0.89 bpp\n\n" + " Supported 3D block sizes are:\n\n" + " 3x3x3: 4.74 bpp 5x5x4: 1.28 bpp\n" + " 4x3x3: 3.56 bpp 5x5x5: 1.02 bpp\n" + " 4x4x3: 2.67 bpp 6x5x5: 0.85 bpp\n" + " 4x4x4: 2.00 bpp 6x6x5: 0.71 bpp\n" + " 5x4x4: 1.60 bpp 6x6x6: 0.59 bpp\n" + " --astc_func \n" + " Specify which transfer function to use while compressing.\n" + " Use this only if you don't trust the color space info in source image.\n" + " Default is what's read from the image.\n" + " --astc_mode \n" + " Specify which encoding mode to use. LDR is the default.\n" + " --astc_quality \n" + " The quality level configures the quality-performance tradeoff for\n" + " the compressor; more complete searches of the search space improve\n" + " image quality at the expense of compression time. Default is 'medium'\n" + " The quality level can be set to fastest (0) and thorough (100) via the \n" + " following fixed quality presets:\n\n" + " fastest (equivalent to quality = 0)\n" + " fast (equivalent to quality = 10)\n" + " medium (equivalent to quality = 60)\n" + " thorough (equivalent to quality = 98)\n" + " exhaustive (equivalent to quality = 100)\n" + " --astc_normal\n" + " The input texture is a three component linear LDR normal map\n" + " storing unit length normals as (R=X, G=Y, B=Z). The output will\n" + " be a two component X+Y normal map stored as (RGB=X, A=Y),\n" + " optimized for angular error instead of simple PSNR. The Z\n" + " component can be recovered programmatically in shader code by\n" + " using the equation:\n\n" + " nml.xy = texture(...).ga; // Load in [0,1]\n" + " nml.xy = nml.xy * 2.0 - 1.0; // Unpack to [-1,1]\n" + " nml.z = sqrt(1 - dot(nml.xy, nml.xy)); // Compute Z\n" + " bcmp:\n" + " Supercompress the image data with ETC1S / BasisLZ.\n" " RED images will become RGB with RED in each component. RG images\n" " will have R in the RGB part and G in the alpha part of the\n" " compressed texture. When set, the following BasisLZ-related\n" @@ -446,26 +445,26 @@ class scApp : public ktxApp { " Disable selector rate distortion optimizations. Slightly faster,\n" " less noisy output, but lower quality per output bit. Default is\n" " to do selector RDO.\n\n" - " uastc:\n" + " uastc:\n" " Create a texture in high-quality transcodable UASTC format.\n" - " --uastc_level \n" - " This optional parameter selects a speed vs quality\n" - " tradeoff as shown in the following table:\n" + " --uastc_level \n" + " This optional parameter selects a speed vs quality\n" + " tradeoff as shown in the following table:\n" "\n" - " Level | Speed | Quality\n" - " ----- | ------- | -------\n" - " 0 | Fastest | 43.45dB\n" - " 1 | Faster | 46.49dB\n" - " 2 | Default | 47.47dB\n" - " 3 | Slower | 48.01dB\n" - " 4 | Very slow | 48.24dB\n" + " Level | Speed | Quality\n" + " ----- | ------- | -------\n" + " 0 | Fastest | 43.45dB\n" + " 1 | Faster | 46.49dB\n" + " 2 | Default | 47.47dB\n" + " 3 | Slower | 48.01dB\n" + " 4 | Very slow | 48.24dB\n" "\n" " You are strongly encouraged to also specify --zcmp to losslessly\n" " compress the UASTC data. This and any LZ-style compression can\n" " be made more effective by conditioning the UASTC texture data\n" " using the Rate Distortion Optimization (RDO) post-process stage.\n" - " When uastc encoding is set the following options become available\n" - " for controlling RDO:\n\n" + " When uastc encoding is set the following options become available\n" + " for controlling RDO:\n\n" " --uastc_rdo_l []\n" " Enable UASTC RDO post-processing and optionally set UASTC RDO\n" " quality scalar (lambda) to @e lambda. Lower values yield higher\n" @@ -496,19 +495,19 @@ class scApp : public ktxApp { " support SSE. Only the Basis Universal compressor uses SSE.\n" " --zcmp []\n" " Supercompress the data with Zstandard. Implies --t2. Can be used\n" - " with data in any format except ETC1S / BasisLZ (bcmp). Most\n" + " with data in any format except ETC1S / BasisLZ (bcmp). Most\n" " effective with RDO-conditioned UASTC or uncompressed formats. The\n" " optional compressionLevel range is 1 - 22 and the default is 3.\n" " Lower values=faster but give less compression. Values above 20\n" - " should be used with caution as they require more memory.\n" - " --threads \n" - " Explicitly set the number of threads to use during compression.\n" - " By default, ETC1S / BasisLZ and astc compression will use the number of\n" - " threads reported by thread::hardware_concurrency or 1 if value\n" - " returned is 0.\n" - " --verbose Print encoder/compressor activity status to stdout. Currently\n" - " only both astcencoder and Basis Universal compressor emits status.\n" - "\n"; + " should be used with caution as they require more memory.\n" + " --threads \n" + " Explicitly set the number of threads to use during compression.\n" + " By default, ETC1S / BasisLZ and ASTC compression will use the number of\n" + " threads reported by thread::hardware_concurrency or 1 if value\n" + " returned is 0.\n" + " --verbose Print encoder/compressor activity status to stdout. Currently\n" + " only both astcencoder and Basis Universal compressor emits status.\n" + "\n"; ktxApp::usage(); cerr << endl << "In case of ambiguity, such as when the last option is one with an optional\n" @@ -525,7 +524,7 @@ scApp::scApp(string& version, string& defaultVersion, : ktxApp(version, defaultVersion, options), options(options) { argparser::option my_option_list[] = { - { "encode", argparser::option::required_argument, NULL, 'b' }, + { "encode", argparser::option::required_argument, NULL, 'b' }, { "zcmp", argparser::option::optional_argument, NULL, 'z' }, { "no_multithreading", argparser::option::no_argument, NULL, 'N' }, { "threads", argparser::option::required_argument, NULL, 't' }, @@ -540,19 +539,19 @@ scApp::scApp(string& version, string& defaultVersion, { "no_endpoint_rdo", argparser::option::no_argument, NULL, 1001 }, { "no_selector_rdo", argparser::option::no_argument, NULL, 1002 }, { "no_sse", argparser::option::no_argument, NULL, 1011 }, - { "uastc_level", argparser::option::required_argument, NULL, 1003 }, + { "uastc_level", argparser::option::required_argument, NULL, 1003 }, { "uastc_rdo_l", argparser::option::optional_argument, NULL, 1004 }, { "uastc_rdo_d", argparser::option::required_argument, NULL, 1005 }, { "uastc_rdo_b", argparser::option::optional_argument, NULL, 1006 }, { "uastc_rdo_s", argparser::option::optional_argument, NULL, 1007 }, { "uastc_rdo_f", argparser::option::no_argument, NULL, 1008 }, { "uastc_rdo_m", argparser::option::no_argument, NULL, 1009 }, - { "verbose", argparser::option::no_argument, NULL, 1010 }, - { "astc_blk_s", argparser::option::required_argument, NULL, 1012 }, - { "astc_func", argparser::option::required_argument, NULL, 1013 }, - { "astc_mode", argparser::option::required_argument, NULL, 1014 }, - { "astc_quality", argparser::option::required_argument, NULL, 1015 }, - { "astc_normal", argparser::option::no_argument, NULL, 1016 } + { "verbose", argparser::option::no_argument, NULL, 1010 }, + { "astc_blk_s", argparser::option::required_argument, NULL, 1012 }, + { "astc_func", argparser::option::required_argument, NULL, 1013 }, + { "astc_mode", argparser::option::required_argument, NULL, 1014 }, + { "astc_quality", argparser::option::required_argument, NULL, 1015 }, + { "astc_normal", argparser::option::no_argument, NULL, 1016 } }; const int lastOptionIndex = sizeof(my_option_list) / sizeof(argparser::option); @@ -599,12 +598,12 @@ scApp::processOption(argparser& parser, int opt) switch (opt) { case 'b': - setEncoder(parser.optarg); + setEncoder(parser.optarg); options.ktx2 = 1; break; case 'z': if (options.bcmp) { - cerr << "Only one of '--encode bcmp' and --zcmp can be specified." + cerr << "Only one of '--encode bcmp' and --zcmp can be specified." << endl; usage(); exit(1); @@ -658,11 +657,11 @@ scApp::processOption(argparser& parser, int opt) break; case 't': options.bopts.threadCount = strtoi(parser.optarg.c_str()); - options.astcopts.threadCount = strtoi(parser.optarg.c_str()); + options.astcopts.threadCount = strtoi(parser.optarg.c_str()); capture = false; break; case 1003: - { + { ktx_uint32_t level = strtoi(parser.optarg.c_str()); level = clamp(level, 0, KTX_PACK_UASTC_MAX_LEVEL); // Ensure the last one wins in case of multiple of these args. @@ -701,32 +700,32 @@ scApp::processOption(argparser& parser, int opt) break; case 1010: options.bopts.verbose = true; - options.astcopts.verbose = true; + options.astcopts.verbose = true; capture = false; break; case 1011: options.bopts.noSSE = true; capture = true; break; - case 1012: // astc_blk_s - options.astcopts.blockSize = getAstcBlockSize(parser.optarg.c_str()); - hasArg = true; - break; - case 1013: // astc_func - options.astcopts.function = getAstcEncoderFunction(parser.optarg.c_str()); - hasArg = true; - break; - case 1014: // astc_mode - options.astcopts.mode = getAstcEncoderMode(parser.optarg.c_str()); - hasArg = true; - break; - case 1015: // astc_quality - options.astcopts.qualityLevel = getAstcQualityLevel(parser.optarg.c_str()); - hasArg = true; - break; - case 1016: // astc_normal - options.astcopts.normalMap = true; - break; + case 1012: // astc_blk_s + options.astcopts.blockSize = astcBlockSize(parser.optarg.c_str()); + hasArg = true; + break; + case 1013: // astc_func + options.astcopts.function = astcEncoderFunction(parser.optarg.c_str()); + hasArg = true; + break; + case 1014: // astc_mode + options.astcopts.mode = astcEncoderMode(parser.optarg.c_str()); + hasArg = true; + break; + case 1015: // astc_quality + options.astcopts.qualityLevel = astcQualityLevel(parser.optarg.c_str()); + hasArg = true; + break; + case 1016: // astc_normal + options.astcopts.normalMap = true; + break; default: return false; } From c01d05d4cfa0353f848c18051d90ba94862115b5 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Sun, 23 May 2021 19:32:16 +0100 Subject: [PATCH 04/65] Fix review comments for astc support --- CMakeLists.txt | 35 +++---- include/ktx.h | 59 ++++++------ lib/astc_encode.cpp | 222 ++++++++++++++++++++++--------------------- lib/basis_encode.cpp | 6 +- utils/scapp.h | 38 ++++---- 5 files changed, 182 insertions(+), 178 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 540e97a98a..4a4b3e380d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -629,25 +629,28 @@ PUBLIC add_subdirectory(interface/basisu_c_binding) -# Add astcenc subdirectory -# Only one architecture is supported at once, if default (avx2) is not available, -# user must define ISA_SSE41, ISA_SSE2 or ISA_NEON themselves +# Only one architecture is supported at once, if neither of +# ISA_SSE41 and ISA_SSE2 are defined ISA_AVX2 is chosen. +# If ISA_AVX2 fails to compile user must chose other x86 options +# On arm based systems ISA_NEON is default -set(ISA_AVX2 ON) # AVX2 is the default -set(ASTC_LIB_NAME astcenc-avx2-static) - -if (${ISA_SSE41}) - set(ASTC_LIB_NAME astcenc-sse4.1-static) -endif() - -if (${ISA_SSE2}) - set(ASTC_LIB_NAME astcenc-sse2-static) -endif() - -if (${ISA_NEON}) - set(ASTC_LIB_NAME astcenc-neon-static) +if(CPU_ARCHITECTURE STREQUAL x86_64 OR CPU_ARCHITECTURE STREQUAL x86) + if (${ISA_SSE41}) + set(ASTC_LIB_NAME astcenc-sse4.1-static) + elseif (${ISA_SSE2}) + set(ASTC_LIB_NAME astcenc-sse2-static) + else() + set(ISA_AVX2 ON) # AVX2 is the default + set(ASTC_LIB_NAME astcenc-avx2-static) + message("ASTCENC: Building for AVX2. If this fails to compile, define either ISA_SSE41 or ISA_SSE2") + endif() +elseif(CPU_ARCHITECTURE STREQUAL arm OR CPU_ARCHITECTURE STREQUAL armv7 + OR CPU_ARCHITECTURE STREQUAL armv8 OR CPU_ARCHITECTURE STREQUAL arm64) + set(ISA_NEON ON) + set(ASTC_LIB_NAME astcenc-neon-static) endif() +# astcenc add_subdirectory(lib/astc-encoder) target_link_libraries(ktx PRIVATE ${ASTC_LIB_NAME}) diff --git a/include/ktx.h b/include/ktx.h index d1439e12f9..7432c549b8 100644 --- a/include/ktx.h +++ b/include/ktx.h @@ -649,9 +649,8 @@ typedef enum ktxSupercmpScheme { KTX_SS_NONE = 0, /*!< No supercompression. */ KTX_SS_BASIS_LZ = 1, /*!< Basis LZ supercompression. */ KTX_SS_ZSTD = 2, /*!< ZStd supercompression. */ - KTX_SS_ASTC = 4, /*!< ASTC compression. */ KTX_SS_BEGIN_RANGE = KTX_SS_NONE, - KTX_SS_END_RANGE = KTX_SS_ASTC, + KTX_SS_END_RANGE = KTX_SS_ZSTD, KTX_SS_BEGIN_VENDOR_RANGE = 0x10000, KTX_SS_END_VENDOR_RANGE = 0x1ffff, KTX_SS_BEGIN_RESERVED = 0x20000, @@ -973,36 +972,36 @@ typedef enum ktx_pack_astc_quality_levels_e { * @~English * @brief Options specifiying ASTC encoding block dimensions */ -typedef enum ktx_pack_astc_block_size_e { +typedef enum ktx_pack_astc_block_dimension_e { // 2D formats - KTX_PACK_ASTC_BLOCK_4x4, //: 8.00 bpp - KTX_PACK_ASTC_BLOCK_5x4, //: 6.40 bpp - KTX_PACK_ASTC_BLOCK_5x5, //: 5.12 bpp - KTX_PACK_ASTC_BLOCK_6x5, //: 4.27 bpp - KTX_PACK_ASTC_BLOCK_6x6, //: 3.56 bpp - KTX_PACK_ASTC_BLOCK_8x5, //: 3.20 bpp - KTX_PACK_ASTC_BLOCK_8x6, //: 2.67 bpp - KTX_PACK_ASTC_BLOCK_10x5, //: 2.56 bpp - KTX_PACK_ASTC_BLOCK_10x6, //: 2.13 bpp - KTX_PACK_ASTC_BLOCK_8x8, //: 2.00 bpp - KTX_PACK_ASTC_BLOCK_10x8, //: 1.60 bpp - KTX_PACK_ASTC_BLOCK_10x10, //: 1.28 bpp - KTX_PACK_ASTC_BLOCK_12x10, //: 1.07 bpp - KTX_PACK_ASTC_BLOCK_12x12, //: 0.89 bpp + KTX_PACK_ASTC_BLOCK_DIMENSION_4x4, //: 8.00 bpp + KTX_PACK_ASTC_BLOCK_DIMENSION_5x4, //: 6.40 bpp + KTX_PACK_ASTC_BLOCK_DIMENSION_5x5, //: 5.12 bpp + KTX_PACK_ASTC_BLOCK_DIMENSION_6x5, //: 4.27 bpp + KTX_PACK_ASTC_BLOCK_DIMENSION_6x6, //: 3.56 bpp + KTX_PACK_ASTC_BLOCK_DIMENSION_8x5, //: 3.20 bpp + KTX_PACK_ASTC_BLOCK_DIMENSION_8x6, //: 2.67 bpp + KTX_PACK_ASTC_BLOCK_DIMENSION_10x5, //: 2.56 bpp + KTX_PACK_ASTC_BLOCK_DIMENSION_10x6, //: 2.13 bpp + KTX_PACK_ASTC_BLOCK_DIMENSION_8x8, //: 2.00 bpp + KTX_PACK_ASTC_BLOCK_DIMENSION_10x8, //: 1.60 bpp + KTX_PACK_ASTC_BLOCK_DIMENSION_10x10, //: 1.28 bpp + KTX_PACK_ASTC_BLOCK_DIMENSION_12x10, //: 1.07 bpp + KTX_PACK_ASTC_BLOCK_DIMENSION_12x12, //: 0.89 bpp // 3D formats - KTX_PACK_ASTC_BLOCK_3x3x3, //: 4.74 bpp - KTX_PACK_ASTC_BLOCK_4x3x3, //: 3.56 bpp - KTX_PACK_ASTC_BLOCK_4x4x3, //: 2.67 bpp - KTX_PACK_ASTC_BLOCK_4x4x4, //: 2.00 bpp - KTX_PACK_ASTC_BLOCK_5x4x4, //: 1.60 bpp - KTX_PACK_ASTC_BLOCK_5x5x4, //: 1.28 bpp - KTX_PACK_ASTC_BLOCK_5x5x5, //: 1.02 bpp - KTX_PACK_ASTC_BLOCK_6x5x5, //: 0.85 bpp - KTX_PACK_ASTC_BLOCK_6x6x5, //: 0.71 bpp - KTX_PACK_ASTC_BLOCK_6x6x6, //: 0.59 bpp - KTX_PACK_ASTC_BLOCK_SIZE_MAX = KTX_PACK_ASTC_BLOCK_6x6x6 + KTX_PACK_ASTC_BLOCK_DIMENSION_3x3x3, //: 4.74 bpp + KTX_PACK_ASTC_BLOCK_DIMENSION_4x3x3, //: 3.56 bpp + KTX_PACK_ASTC_BLOCK_DIMENSION_4x4x3, //: 2.67 bpp + KTX_PACK_ASTC_BLOCK_DIMENSION_4x4x4, //: 2.00 bpp + KTX_PACK_ASTC_BLOCK_DIMENSION_5x4x4, //: 1.60 bpp + KTX_PACK_ASTC_BLOCK_DIMENSION_5x5x4, //: 1.28 bpp + KTX_PACK_ASTC_BLOCK_DIMENSION_5x5x5, //: 1.02 bpp + KTX_PACK_ASTC_BLOCK_DIMENSION_6x5x5, //: 0.85 bpp + KTX_PACK_ASTC_BLOCK_DIMENSION_6x6x5, //: 0.71 bpp + KTX_PACK_ASTC_BLOCK_DIMENSION_6x6x6, //: 0.59 bpp + KTX_PACK_ASTC_BLOCK_DIMENSION_MAX = KTX_PACK_ASTC_BLOCK_DIMENSION_6x6x6 /*!< Maximum supported blocks. */ -} ktx_pack_astc_block_size_e; +} ktx_pack_astc_block_dimension_e; /** * @~English @@ -1050,7 +1049,7 @@ typedef struct ktxAstcParams { /*!< Number of threads used for compression. Default is 1. */ /* astcenc params */ - ktx_uint32_t blockSize; + ktx_uint32_t blockDimension; /*!< Combinations of block dimensions that astcenc supports i.e. 6x6, 8x8, 6x5 etc*/ diff --git a/lib/astc_encode.cpp b/lib/astc_encode.cpp index 31305e2fc1..246e23860c 100644 --- a/lib/astc_encode.cpp +++ b/lib/astc_encode.cpp @@ -223,7 +223,7 @@ astcDefaultOptions() { params.structSize = sizeof(params); params.verbose = false; params.threadCount = 1; - params.blockSize = KTX_PACK_ASTC_BLOCK_6x6; + params.blockDimension = KTX_PACK_ASTC_BLOCK_DIMENSION_6x6; params.function = KTX_PACK_ASTC_ENCODER_FUNCTION_UNKNOWN; params.mode = KTX_PACK_ASTC_ENCODER_MODE_LDR; params.qualityLevel = KTX_PACK_ASTC_QUALITY_LEVEL_MEDIUM; @@ -236,44 +236,44 @@ astcDefaultOptions() { * @memberof ktxTexture * @ingroup write * @~English - * @brief Creates valid astc block size from string. + * @brief Creates valid astc block dimension from string. * - * @return Valid ktx_pack_astc_block_size_e from string + * @return Valid ktx_pack_astc_block_dimension_e from string */ -ktx_pack_astc_block_size_e -astcBlockSize(const char* block_size) { - static std::unordered_map - astc_blocks_mapping{{"4x4", KTX_PACK_ASTC_BLOCK_4x4}, - {"5x4", KTX_PACK_ASTC_BLOCK_5x4}, - {"5x5", KTX_PACK_ASTC_BLOCK_5x5}, - {"6x5", KTX_PACK_ASTC_BLOCK_6x5}, - {"6x6", KTX_PACK_ASTC_BLOCK_6x6}, - {"8x5", KTX_PACK_ASTC_BLOCK_8x5}, - {"8x6", KTX_PACK_ASTC_BLOCK_8x6}, - {"10x5", KTX_PACK_ASTC_BLOCK_10x5}, - {"10x6", KTX_PACK_ASTC_BLOCK_10x6}, - {"8x8", KTX_PACK_ASTC_BLOCK_8x8}, - {"10x8", KTX_PACK_ASTC_BLOCK_10x8}, - {"10x10", KTX_PACK_ASTC_BLOCK_10x10}, - {"12x10", KTX_PACK_ASTC_BLOCK_12x10}, - {"12x12", KTX_PACK_ASTC_BLOCK_12x12}, - {"3x3x3", KTX_PACK_ASTC_BLOCK_3x3x3}, - {"4x3x3", KTX_PACK_ASTC_BLOCK_4x3x3}, - {"4x4x3", KTX_PACK_ASTC_BLOCK_4x4x3}, - {"4x4x4", KTX_PACK_ASTC_BLOCK_4x4x4}, - {"5x4x4", KTX_PACK_ASTC_BLOCK_5x4x4}, - {"5x5x4", KTX_PACK_ASTC_BLOCK_5x5x4}, - {"5x5x5", KTX_PACK_ASTC_BLOCK_5x5x5}, - {"6x5x5", KTX_PACK_ASTC_BLOCK_6x5x5}, - {"6x6x5", KTX_PACK_ASTC_BLOCK_6x6x5}, - {"6x6x6", KTX_PACK_ASTC_BLOCK_6x6x6}}; +ktx_pack_astc_block_dimension_e +astcBlockDimension(const char* block_size) { + static std::unordered_map + astc_blocks_mapping{{"4x4", KTX_PACK_ASTC_BLOCK_DIMENSION_4x4}, + {"5x4", KTX_PACK_ASTC_BLOCK_DIMENSION_5x4}, + {"5x5", KTX_PACK_ASTC_BLOCK_DIMENSION_5x5}, + {"6x5", KTX_PACK_ASTC_BLOCK_DIMENSION_6x5}, + {"6x6", KTX_PACK_ASTC_BLOCK_DIMENSION_6x6}, + {"8x5", KTX_PACK_ASTC_BLOCK_DIMENSION_8x5}, + {"8x6", KTX_PACK_ASTC_BLOCK_DIMENSION_8x6}, + {"10x5", KTX_PACK_ASTC_BLOCK_DIMENSION_10x5}, + {"10x6", KTX_PACK_ASTC_BLOCK_DIMENSION_10x6}, + {"8x8", KTX_PACK_ASTC_BLOCK_DIMENSION_8x8}, + {"10x8", KTX_PACK_ASTC_BLOCK_DIMENSION_10x8}, + {"10x10", KTX_PACK_ASTC_BLOCK_DIMENSION_10x10}, + {"12x10", KTX_PACK_ASTC_BLOCK_DIMENSION_12x10}, + {"12x12", KTX_PACK_ASTC_BLOCK_DIMENSION_12x12}, + {"3x3x3", KTX_PACK_ASTC_BLOCK_DIMENSION_3x3x3}, + {"4x3x3", KTX_PACK_ASTC_BLOCK_DIMENSION_4x3x3}, + {"4x4x3", KTX_PACK_ASTC_BLOCK_DIMENSION_4x4x3}, + {"4x4x4", KTX_PACK_ASTC_BLOCK_DIMENSION_4x4x4}, + {"5x4x4", KTX_PACK_ASTC_BLOCK_DIMENSION_5x4x4}, + {"5x5x4", KTX_PACK_ASTC_BLOCK_DIMENSION_5x5x4}, + {"5x5x5", KTX_PACK_ASTC_BLOCK_DIMENSION_5x5x5}, + {"6x5x5", KTX_PACK_ASTC_BLOCK_DIMENSION_6x5x5}, + {"6x6x5", KTX_PACK_ASTC_BLOCK_DIMENSION_6x6x5}, + {"6x6x6", KTX_PACK_ASTC_BLOCK_DIMENSION_6x6x6}}; auto opt = astc_blocks_mapping.find(block_size); if (opt != astc_blocks_mapping.end()) return opt->second; - return KTX_PACK_ASTC_BLOCK_6x6; + return KTX_PACK_ASTC_BLOCK_DIMENSION_6x6; } /** @@ -353,57 +353,57 @@ static ktx_uint32_t astcVkFormat(ktx_uint32_t block_size, bool sRGB) { if (sRGB) { switch (block_size) { - case KTX_PACK_ASTC_BLOCK_4x4: return VK_FORMAT_ASTC_4x4_SRGB_BLOCK; - case KTX_PACK_ASTC_BLOCK_5x4: return VK_FORMAT_ASTC_5x4_SRGB_BLOCK; - case KTX_PACK_ASTC_BLOCK_5x5: return VK_FORMAT_ASTC_5x5_SRGB_BLOCK; - case KTX_PACK_ASTC_BLOCK_6x5: return VK_FORMAT_ASTC_6x5_SRGB_BLOCK; - case KTX_PACK_ASTC_BLOCK_6x6: return VK_FORMAT_ASTC_6x6_SRGB_BLOCK; - case KTX_PACK_ASTC_BLOCK_8x5: return VK_FORMAT_ASTC_8x5_SRGB_BLOCK; - case KTX_PACK_ASTC_BLOCK_8x6: return VK_FORMAT_ASTC_8x6_SRGB_BLOCK; - case KTX_PACK_ASTC_BLOCK_8x8: return VK_FORMAT_ASTC_8x8_SRGB_BLOCK; - case KTX_PACK_ASTC_BLOCK_10x5: return VK_FORMAT_ASTC_10x5_SRGB_BLOCK; - case KTX_PACK_ASTC_BLOCK_10x6: return VK_FORMAT_ASTC_10x6_SRGB_BLOCK; - case KTX_PACK_ASTC_BLOCK_10x8: return VK_FORMAT_ASTC_10x8_SRGB_BLOCK; - case KTX_PACK_ASTC_BLOCK_10x10: return VK_FORMAT_ASTC_10x10_SRGB_BLOCK; - case KTX_PACK_ASTC_BLOCK_12x10: return VK_FORMAT_ASTC_12x10_SRGB_BLOCK; - case KTX_PACK_ASTC_BLOCK_12x12: return VK_FORMAT_ASTC_12x12_SRGB_BLOCK; - case KTX_PACK_ASTC_BLOCK_3x3x3: return VK_FORMAT_ASTC_3x3x3_SRGB_BLOCK_EXT; - case KTX_PACK_ASTC_BLOCK_4x3x3: return VK_FORMAT_ASTC_4x3x3_SRGB_BLOCK_EXT; - case KTX_PACK_ASTC_BLOCK_4x4x3: return VK_FORMAT_ASTC_4x4x3_SRGB_BLOCK_EXT; - case KTX_PACK_ASTC_BLOCK_4x4x4: return VK_FORMAT_ASTC_4x4x4_SRGB_BLOCK_EXT; - case KTX_PACK_ASTC_BLOCK_5x4x4: return VK_FORMAT_ASTC_5x4x4_SRGB_BLOCK_EXT; - case KTX_PACK_ASTC_BLOCK_5x5x4: return VK_FORMAT_ASTC_5x5x4_SRGB_BLOCK_EXT; - case KTX_PACK_ASTC_BLOCK_5x5x5: return VK_FORMAT_ASTC_5x5x5_SRGB_BLOCK_EXT; - case KTX_PACK_ASTC_BLOCK_6x5x5: return VK_FORMAT_ASTC_6x5x5_SRGB_BLOCK_EXT; - case KTX_PACK_ASTC_BLOCK_6x6x5: return VK_FORMAT_ASTC_6x6x5_SRGB_BLOCK_EXT; - case KTX_PACK_ASTC_BLOCK_6x6x6: return VK_FORMAT_ASTC_6x6x6_SRGB_BLOCK_EXT; + case KTX_PACK_ASTC_BLOCK_DIMENSION_4x4: return VK_FORMAT_ASTC_4x4_SRGB_BLOCK; + case KTX_PACK_ASTC_BLOCK_DIMENSION_5x4: return VK_FORMAT_ASTC_5x4_SRGB_BLOCK; + case KTX_PACK_ASTC_BLOCK_DIMENSION_5x5: return VK_FORMAT_ASTC_5x5_SRGB_BLOCK; + case KTX_PACK_ASTC_BLOCK_DIMENSION_6x5: return VK_FORMAT_ASTC_6x5_SRGB_BLOCK; + case KTX_PACK_ASTC_BLOCK_DIMENSION_6x6: return VK_FORMAT_ASTC_6x6_SRGB_BLOCK; + case KTX_PACK_ASTC_BLOCK_DIMENSION_8x5: return VK_FORMAT_ASTC_8x5_SRGB_BLOCK; + case KTX_PACK_ASTC_BLOCK_DIMENSION_8x6: return VK_FORMAT_ASTC_8x6_SRGB_BLOCK; + case KTX_PACK_ASTC_BLOCK_DIMENSION_8x8: return VK_FORMAT_ASTC_8x8_SRGB_BLOCK; + case KTX_PACK_ASTC_BLOCK_DIMENSION_10x5: return VK_FORMAT_ASTC_10x5_SRGB_BLOCK; + case KTX_PACK_ASTC_BLOCK_DIMENSION_10x6: return VK_FORMAT_ASTC_10x6_SRGB_BLOCK; + case KTX_PACK_ASTC_BLOCK_DIMENSION_10x8: return VK_FORMAT_ASTC_10x8_SRGB_BLOCK; + case KTX_PACK_ASTC_BLOCK_DIMENSION_10x10: return VK_FORMAT_ASTC_10x10_SRGB_BLOCK; + case KTX_PACK_ASTC_BLOCK_DIMENSION_12x10: return VK_FORMAT_ASTC_12x10_SRGB_BLOCK; + case KTX_PACK_ASTC_BLOCK_DIMENSION_12x12: return VK_FORMAT_ASTC_12x12_SRGB_BLOCK; + case KTX_PACK_ASTC_BLOCK_DIMENSION_3x3x3: return VK_FORMAT_ASTC_3x3x3_SRGB_BLOCK_EXT; + case KTX_PACK_ASTC_BLOCK_DIMENSION_4x3x3: return VK_FORMAT_ASTC_4x3x3_SRGB_BLOCK_EXT; + case KTX_PACK_ASTC_BLOCK_DIMENSION_4x4x3: return VK_FORMAT_ASTC_4x4x3_SRGB_BLOCK_EXT; + case KTX_PACK_ASTC_BLOCK_DIMENSION_4x4x4: return VK_FORMAT_ASTC_4x4x4_SRGB_BLOCK_EXT; + case KTX_PACK_ASTC_BLOCK_DIMENSION_5x4x4: return VK_FORMAT_ASTC_5x4x4_SRGB_BLOCK_EXT; + case KTX_PACK_ASTC_BLOCK_DIMENSION_5x5x4: return VK_FORMAT_ASTC_5x5x4_SRGB_BLOCK_EXT; + case KTX_PACK_ASTC_BLOCK_DIMENSION_5x5x5: return VK_FORMAT_ASTC_5x5x5_SRGB_BLOCK_EXT; + case KTX_PACK_ASTC_BLOCK_DIMENSION_6x5x5: return VK_FORMAT_ASTC_6x5x5_SRGB_BLOCK_EXT; + case KTX_PACK_ASTC_BLOCK_DIMENSION_6x6x5: return VK_FORMAT_ASTC_6x6x5_SRGB_BLOCK_EXT; + case KTX_PACK_ASTC_BLOCK_DIMENSION_6x6x6: return VK_FORMAT_ASTC_6x6x6_SRGB_BLOCK_EXT; } } else { switch (block_size) { - case KTX_PACK_ASTC_BLOCK_4x4: return VK_FORMAT_ASTC_4x4_UNORM_BLOCK; - case KTX_PACK_ASTC_BLOCK_5x4: return VK_FORMAT_ASTC_5x4_UNORM_BLOCK; - case KTX_PACK_ASTC_BLOCK_5x5: return VK_FORMAT_ASTC_5x5_UNORM_BLOCK; - case KTX_PACK_ASTC_BLOCK_6x5: return VK_FORMAT_ASTC_6x5_UNORM_BLOCK; - case KTX_PACK_ASTC_BLOCK_6x6: return VK_FORMAT_ASTC_6x6_UNORM_BLOCK; - case KTX_PACK_ASTC_BLOCK_8x5: return VK_FORMAT_ASTC_8x5_UNORM_BLOCK; - case KTX_PACK_ASTC_BLOCK_8x6: return VK_FORMAT_ASTC_8x6_UNORM_BLOCK; - case KTX_PACK_ASTC_BLOCK_8x8: return VK_FORMAT_ASTC_8x8_UNORM_BLOCK; - case KTX_PACK_ASTC_BLOCK_10x5: return VK_FORMAT_ASTC_10x5_UNORM_BLOCK; - case KTX_PACK_ASTC_BLOCK_10x6: return VK_FORMAT_ASTC_10x6_UNORM_BLOCK; - case KTX_PACK_ASTC_BLOCK_10x8: return VK_FORMAT_ASTC_10x8_UNORM_BLOCK; - case KTX_PACK_ASTC_BLOCK_10x10: return VK_FORMAT_ASTC_10x10_UNORM_BLOCK; - case KTX_PACK_ASTC_BLOCK_12x10: return VK_FORMAT_ASTC_12x10_UNORM_BLOCK; - case KTX_PACK_ASTC_BLOCK_12x12: return VK_FORMAT_ASTC_12x12_UNORM_BLOCK; - case KTX_PACK_ASTC_BLOCK_3x3x3: return VK_FORMAT_ASTC_3x3x3_UNORM_BLOCK_EXT; - case KTX_PACK_ASTC_BLOCK_4x3x3: return VK_FORMAT_ASTC_4x3x3_UNORM_BLOCK_EXT; - case KTX_PACK_ASTC_BLOCK_4x4x3: return VK_FORMAT_ASTC_4x4x3_UNORM_BLOCK_EXT; - case KTX_PACK_ASTC_BLOCK_4x4x4: return VK_FORMAT_ASTC_4x4x4_UNORM_BLOCK_EXT; - case KTX_PACK_ASTC_BLOCK_5x4x4: return VK_FORMAT_ASTC_5x4x4_UNORM_BLOCK_EXT; - case KTX_PACK_ASTC_BLOCK_5x5x4: return VK_FORMAT_ASTC_5x5x4_UNORM_BLOCK_EXT; - case KTX_PACK_ASTC_BLOCK_5x5x5: return VK_FORMAT_ASTC_5x5x5_UNORM_BLOCK_EXT; - case KTX_PACK_ASTC_BLOCK_6x5x5: return VK_FORMAT_ASTC_6x5x5_UNORM_BLOCK_EXT; - case KTX_PACK_ASTC_BLOCK_6x6x5: return VK_FORMAT_ASTC_6x6x5_UNORM_BLOCK_EXT; - case KTX_PACK_ASTC_BLOCK_6x6x6: return VK_FORMAT_ASTC_6x6x6_UNORM_BLOCK_EXT; + case KTX_PACK_ASTC_BLOCK_DIMENSION_4x4: return VK_FORMAT_ASTC_4x4_UNORM_BLOCK; + case KTX_PACK_ASTC_BLOCK_DIMENSION_5x4: return VK_FORMAT_ASTC_5x4_UNORM_BLOCK; + case KTX_PACK_ASTC_BLOCK_DIMENSION_5x5: return VK_FORMAT_ASTC_5x5_UNORM_BLOCK; + case KTX_PACK_ASTC_BLOCK_DIMENSION_6x5: return VK_FORMAT_ASTC_6x5_UNORM_BLOCK; + case KTX_PACK_ASTC_BLOCK_DIMENSION_6x6: return VK_FORMAT_ASTC_6x6_UNORM_BLOCK; + case KTX_PACK_ASTC_BLOCK_DIMENSION_8x5: return VK_FORMAT_ASTC_8x5_UNORM_BLOCK; + case KTX_PACK_ASTC_BLOCK_DIMENSION_8x6: return VK_FORMAT_ASTC_8x6_UNORM_BLOCK; + case KTX_PACK_ASTC_BLOCK_DIMENSION_8x8: return VK_FORMAT_ASTC_8x8_UNORM_BLOCK; + case KTX_PACK_ASTC_BLOCK_DIMENSION_10x5: return VK_FORMAT_ASTC_10x5_UNORM_BLOCK; + case KTX_PACK_ASTC_BLOCK_DIMENSION_10x6: return VK_FORMAT_ASTC_10x6_UNORM_BLOCK; + case KTX_PACK_ASTC_BLOCK_DIMENSION_10x8: return VK_FORMAT_ASTC_10x8_UNORM_BLOCK; + case KTX_PACK_ASTC_BLOCK_DIMENSION_10x10: return VK_FORMAT_ASTC_10x10_UNORM_BLOCK; + case KTX_PACK_ASTC_BLOCK_DIMENSION_12x10: return VK_FORMAT_ASTC_12x10_UNORM_BLOCK; + case KTX_PACK_ASTC_BLOCK_DIMENSION_12x12: return VK_FORMAT_ASTC_12x12_UNORM_BLOCK; + case KTX_PACK_ASTC_BLOCK_DIMENSION_3x3x3: return VK_FORMAT_ASTC_3x3x3_UNORM_BLOCK_EXT; + case KTX_PACK_ASTC_BLOCK_DIMENSION_4x3x3: return VK_FORMAT_ASTC_4x3x3_UNORM_BLOCK_EXT; + case KTX_PACK_ASTC_BLOCK_DIMENSION_4x4x3: return VK_FORMAT_ASTC_4x4x3_UNORM_BLOCK_EXT; + case KTX_PACK_ASTC_BLOCK_DIMENSION_4x4x4: return VK_FORMAT_ASTC_4x4x4_UNORM_BLOCK_EXT; + case KTX_PACK_ASTC_BLOCK_DIMENSION_5x4x4: return VK_FORMAT_ASTC_5x4x4_UNORM_BLOCK_EXT; + case KTX_PACK_ASTC_BLOCK_DIMENSION_5x5x4: return VK_FORMAT_ASTC_5x5x4_UNORM_BLOCK_EXT; + case KTX_PACK_ASTC_BLOCK_DIMENSION_5x5x5: return VK_FORMAT_ASTC_5x5x5_UNORM_BLOCK_EXT; + case KTX_PACK_ASTC_BLOCK_DIMENSION_6x5x5: return VK_FORMAT_ASTC_6x5x5_UNORM_BLOCK_EXT; + case KTX_PACK_ASTC_BLOCK_DIMENSION_6x6x5: return VK_FORMAT_ASTC_6x6x5_UNORM_BLOCK_EXT; + case KTX_PACK_ASTC_BLOCK_DIMENSION_6x6x6: return VK_FORMAT_ASTC_6x6x6_UNORM_BLOCK_EXT; } } @@ -489,33 +489,33 @@ astcSwizzle(const ktxAstcParams ¶ms) { } static void -astcBlockSizes(ktx_uint32_t block_size, +astcBlockDimensions(ktx_uint32_t block_size, uint32_t& block_x, uint32_t& block_y, uint32_t& block_z) { switch (block_size) { - case KTX_PACK_ASTC_BLOCK_4x4 : block_x = 4; block_y = 4; block_z = 1; break; - case KTX_PACK_ASTC_BLOCK_5x4 : block_x = 5; block_y = 4; block_z = 1; break; - case KTX_PACK_ASTC_BLOCK_5x5 : block_x = 5; block_y = 5; block_z = 1; break; - case KTX_PACK_ASTC_BLOCK_6x5 : block_x = 6; block_y = 5; block_z = 1; break; - case KTX_PACK_ASTC_BLOCK_6x6 : block_x = 6; block_y = 6; block_z = 1; break; - case KTX_PACK_ASTC_BLOCK_8x5 : block_x = 8; block_y = 5; block_z = 1; break; - case KTX_PACK_ASTC_BLOCK_8x6 : block_x = 8; block_y = 6; block_z = 1; break; - case KTX_PACK_ASTC_BLOCK_10x5 : block_x = 10; block_y = 5; block_z = 1; break; - case KTX_PACK_ASTC_BLOCK_10x6 : block_x = 10; block_y = 6; block_z = 1; break; - case KTX_PACK_ASTC_BLOCK_8x8 : block_x = 8; block_y = 8; block_z = 1; break; - case KTX_PACK_ASTC_BLOCK_10x8 : block_x = 10; block_y = 8; block_z = 1; break; - case KTX_PACK_ASTC_BLOCK_10x10 : block_x = 10; block_y = 10; block_z = 1; break; - case KTX_PACK_ASTC_BLOCK_12x10 : block_x = 12; block_y = 10; block_z = 1; break; - case KTX_PACK_ASTC_BLOCK_12x12 : block_x = 12; block_y = 12; block_z = 1; break; - case KTX_PACK_ASTC_BLOCK_3x3x3 : block_x = 3; block_y = 3; block_z = 3; break; - case KTX_PACK_ASTC_BLOCK_4x3x3 : block_x = 4; block_y = 3; block_z = 3; break; - case KTX_PACK_ASTC_BLOCK_4x4x3 : block_x = 4; block_y = 4; block_z = 3; break; - case KTX_PACK_ASTC_BLOCK_4x4x4 : block_x = 4; block_y = 4; block_z = 4; break; - case KTX_PACK_ASTC_BLOCK_5x4x4 : block_x = 5; block_y = 4; block_z = 4; break; - case KTX_PACK_ASTC_BLOCK_5x5x4 : block_x = 5; block_y = 5; block_z = 4; break; - case KTX_PACK_ASTC_BLOCK_5x5x5 : block_x = 5; block_y = 5; block_z = 5; break; - case KTX_PACK_ASTC_BLOCK_6x5x5 : block_x = 6; block_y = 5; block_z = 5; break; - case KTX_PACK_ASTC_BLOCK_6x6x5 : block_x = 6; block_y = 6; block_z = 5; break; - case KTX_PACK_ASTC_BLOCK_6x6x6 : block_x = 6; block_y = 6; block_z = 6; break; + case KTX_PACK_ASTC_BLOCK_DIMENSION_4x4 : block_x = 4; block_y = 4; block_z = 1; break; + case KTX_PACK_ASTC_BLOCK_DIMENSION_5x4 : block_x = 5; block_y = 4; block_z = 1; break; + case KTX_PACK_ASTC_BLOCK_DIMENSION_5x5 : block_x = 5; block_y = 5; block_z = 1; break; + case KTX_PACK_ASTC_BLOCK_DIMENSION_6x5 : block_x = 6; block_y = 5; block_z = 1; break; + case KTX_PACK_ASTC_BLOCK_DIMENSION_6x6 : block_x = 6; block_y = 6; block_z = 1; break; + case KTX_PACK_ASTC_BLOCK_DIMENSION_8x5 : block_x = 8; block_y = 5; block_z = 1; break; + case KTX_PACK_ASTC_BLOCK_DIMENSION_8x6 : block_x = 8; block_y = 6; block_z = 1; break; + case KTX_PACK_ASTC_BLOCK_DIMENSION_10x5 : block_x = 10; block_y = 5; block_z = 1; break; + case KTX_PACK_ASTC_BLOCK_DIMENSION_10x6 : block_x = 10; block_y = 6; block_z = 1; break; + case KTX_PACK_ASTC_BLOCK_DIMENSION_8x8 : block_x = 8; block_y = 8; block_z = 1; break; + case KTX_PACK_ASTC_BLOCK_DIMENSION_10x8 : block_x = 10; block_y = 8; block_z = 1; break; + case KTX_PACK_ASTC_BLOCK_DIMENSION_10x10 : block_x = 10; block_y = 10; block_z = 1; break; + case KTX_PACK_ASTC_BLOCK_DIMENSION_12x10 : block_x = 12; block_y = 10; block_z = 1; break; + case KTX_PACK_ASTC_BLOCK_DIMENSION_12x12 : block_x = 12; block_y = 12; block_z = 1; break; + case KTX_PACK_ASTC_BLOCK_DIMENSION_3x3x3 : block_x = 3; block_y = 3; block_z = 3; break; + case KTX_PACK_ASTC_BLOCK_DIMENSION_4x3x3 : block_x = 4; block_y = 3; block_z = 3; break; + case KTX_PACK_ASTC_BLOCK_DIMENSION_4x4x3 : block_x = 4; block_y = 4; block_z = 3; break; + case KTX_PACK_ASTC_BLOCK_DIMENSION_4x4x4 : block_x = 4; block_y = 4; block_z = 4; break; + case KTX_PACK_ASTC_BLOCK_DIMENSION_5x4x4 : block_x = 5; block_y = 4; block_z = 4; break; + case KTX_PACK_ASTC_BLOCK_DIMENSION_5x5x4 : block_x = 5; block_y = 5; block_z = 4; break; + case KTX_PACK_ASTC_BLOCK_DIMENSION_5x5x5 : block_x = 5; block_y = 5; block_z = 5; break; + case KTX_PACK_ASTC_BLOCK_DIMENSION_6x5x5 : block_x = 6; block_y = 5; block_z = 5; break; + case KTX_PACK_ASTC_BLOCK_DIMENSION_6x6x5 : block_x = 6; block_y = 6; block_z = 5; break; + case KTX_PACK_ASTC_BLOCK_DIMENSION_6x6x6 : block_x = 6; block_y = 6; block_z = 6; break; default: block_x = 6; block_y = 6; block_z = 1; break; } @@ -637,7 +637,7 @@ launchThreads(int threadCount, void (*func)(int, int, void*), void *payload) { * @return KTX_SUCCESS on success, other KTX_* enum values on error. * * @exception KTX_INVALID_OPERATION - * The texture is already supercompressed. + * The texture's images are supercompressed. * @exception KTX_INVALID_OPERATION * The texture's image are in a block compressed * format. @@ -653,7 +653,7 @@ launchThreads(int threadCount, void (*func)(int, int, void*), void *payload) { * @exception KTX_INVALID_OPERATION * Astc compressor failed to compress image for any reason. - * @exception KTX_OUT_OF_MEMORY Not enough memory to carry out supercompression. + * @exception KTX_OUT_OF_MEMORY Not enough memory to carry out compression. */ extern "C" KTX_error_code ktxTexture_CompressAstcEx(ktxTexture* _This, ktxAstcParams* params) { @@ -709,7 +709,7 @@ ktxTexture_CompressAstcEx(ktxTexture* _This, ktxAstcParams* params) { float quality{ASTCENC_PRE_MEDIUM}; uint32_t flags{params->normalMap ? ASTCENC_FLG_MAP_NORMAL : 0}; - astcBlockSizes(params->blockSize, + astcBlockDimensions(params->blockDimension, block_size_x, block_size_y, block_size_z); quality = astcQuality(params->qualityLevel); profile = astcEncoderAction(*params, BDB); @@ -757,6 +757,9 @@ ktxTexture_CompressAstcEx(ktxTexture* _This, ktxAstcParams* params) { // Walk in reverse on levels so we don't have to do this later // This->numLevels = 0 not allowed for block compressed formats + // But just in case make sure its not zero + This->numLevels = MAX(1, This->numLevels); + for (int32_t level = This->numLevels - 1; level >= 0; level--) { uint32_t width = MAX(1, This->baseWidth >> level); uint32_t height = MAX(1, This->baseHeight >> level); @@ -876,7 +879,7 @@ ktxTexture_CompressAstcEx(ktxTexture* _This, ktxAstcParams* params) { ktx_uint32_t transfer = KHR_DFDVAL(BDB, TRANSFER); bool sRGB = transfer == KHR_DF_TRANSFER_SRGB; // Is this right or I need "KHR_DF_PRIMARIES_SRGB"? - This->vkFormat = astcVkFormat(params->blockSize, sRGB); + This->vkFormat = astcVkFormat(params->blockDimension, sRGB); free(This->pDfd); This->pDfd = vk2dfd(static_cast(This->vkFormat)); @@ -887,7 +890,6 @@ ktxTexture_CompressAstcEx(ktxTexture* _This, ktxAstcParams* params) { // Astc-related checks ktx_uint32_t model = KHR_DFDVAL(BDB, MODEL); ktx_uint32_t primaries = KHR_DFDVAL(BDB, PRIMARIES); - This->supercompressionScheme = KTX_SS_ASTC; This->isCompressed = true; assert(model == KHR_DF_MODEL_ASTC && "Invalid dfd generated for astc image\n"); @@ -895,7 +897,7 @@ ktxTexture_CompressAstcEx(ktxTexture* _This, ktxAstcParams* params) { assert(primaries == KHR_DF_PRIMARIES_SRGB && "Not a valid sRGB image\n"); } - // Since we only allow 8-bit components to be compressed + // Block-compressed textures never need byte swapping so typeSize is 1. This->_protected->_typeSize = 1; return KTX_SUCCESS; diff --git a/lib/basis_encode.cpp b/lib/basis_encode.cpp index 14b6430cb7..038a391775 100644 --- a/lib/basis_encode.cpp +++ b/lib/basis_encode.cpp @@ -379,7 +379,7 @@ static bool basisuEncoderInitialized = false; * @return KTX_SUCCESS on success, other KTX_* enum values on error. * * @exception KTX_INVALID_OPERATION - * The texture is already supercompressed. + * The texture's images are supercompressed. * @exception KTX_INVALID_OPERATION * The texture's image are in a block compressed * format. @@ -397,7 +397,7 @@ static bool basisuEncoderInitialized = false; * @exception KTX_INVALID_OPERATION * Both preSwizzle and and inputSwizzle are specified * in @a params. - * @exception KTX_OUT_OF_MEMORY Not enough memory to carry out supercompression. + * @exception KTX_OUT_OF_MEMORY Not enough memory to carry out compression. */ extern "C" KTX_error_code ktxTexture2_CompressBasisEx(ktxTexture2* This, ktxBasisParams* params) @@ -994,7 +994,7 @@ ktxTexture2_CompressBasisEx(ktxTexture2* This, ktxBasisParams* params) } This->vkFormat = VK_FORMAT_UNDEFINED; - // Since we only allow 8-bit components to be compressed ... + // Block-compressed textures never need byte swapping so typeSize is 1. assert(This->_protected->_typeSize == 1); // Copy in the compressed image data. diff --git a/utils/scapp.h b/utils/scapp.h index 69c105efa8..3867a84982 100644 --- a/utils/scapp.h +++ b/utils/scapp.h @@ -38,8 +38,8 @@ struct clampedOption T max; }; -ktx_pack_astc_block_size_e -astcBlockSize(const char* block_size); +ktx_pack_astc_block_dimension_e +astcBlockDimension(const char* block_dimension); ktx_pack_astc_quality_levels_e astcQualityLevel(const char* quality); @@ -260,14 +260,14 @@ class scApp : public ktxApp { struct astcOptions : public ktxAstcParams { clampedOption threadCount; - clampedOption blockSize; + clampedOption blockDimension; clampedOption function; clampedOption mode; clampedOption qualityLevel; astcOptions() : threadCount(ktxAstcParams::threadCount, 1, 10000), - blockSize(ktxAstcParams::blockSize, 0, KTX_PACK_ASTC_BLOCK_SIZE_MAX), + blockDimension(ktxAstcParams::blockDimension, 0, KTX_PACK_ASTC_BLOCK_DIMENSION_MAX), function(ktxAstcParams::function, 0, KTX_PACK_ASTC_ENCODER_FUNCTION_MAX), mode(ktxAstcParams::mode, 0, KTX_PACK_ASTC_ENCODER_MODE_MAX), qualityLevel(ktxAstcParams::qualityLevel, 0, KTX_PACK_ASTC_QUALITY_LEVEL_MAX) @@ -278,8 +278,8 @@ class scApp : public ktxApp { threadCount = tc; structSize = sizeof(ktxAstcParams); - blockSize.clear(); - blockSize = KTX_PACK_ASTC_BLOCK_6x6; + blockDimension.clear(); + blockDimension = KTX_PACK_ASTC_BLOCK_DIMENSION_6x6; function.clear(); // Default to unknown to have a chance to use to use color space from file function = KTX_PACK_ASTC_ENCODER_FUNCTION_UNKNOWN; @@ -336,16 +336,16 @@ class scApp : public ktxApp { " --encode \n" " Compress the image data with ETC1S / BasisLZ, high-quality transcodable\n" " UASTC format or ASTC format. Implies --t2 for all encoding.\n" - " Accepable options are 'bcmp', 'uastc' and 'astc'. With each encoding\n" + " Acceptable options are 'bcmp', 'uastc' and 'astc'. With each encoding\n" " option the following encoding specific options become valid, otherwise.\n" " they are ignored.\n\n" " astc:\n" " Create a texture in high-quality ASTC format.\n" - " --astc_blk_s /\n" - " Specify which block size to use for compressing the textures.\n" - " e.g. --astc_blk_s 6x5 for 2D or --astc_blk_s 6x6x6 for 3D.\n" + " --astc_blk_d /\n" + " Specify which block dimension to use for compressing the textures.\n" + " e.g. --astc_blk_d 6x5 for 2D or --astc_blk_d 6x6x6 for 3D.\n" " 6x6 is default for 2D.\n\n" - " Supported 2D block sizes are:\n\n" + " Supported 2D block dimensions are:\n\n" " 4x4: 8.00 bpp 10x5: 2.56 bpp\n" " 5x4: 6.40 bpp 10x6: 2.13 bpp\n" " 5x5: 5.12 bpp 8x8: 2.00 bpp\n" @@ -353,7 +353,7 @@ class scApp : public ktxApp { " 6x6: 3.56 bpp 10x10: 1.28 bpp\n" " 8x5: 3.20 bpp 12x10: 1.07 bpp\n" " 8x6: 2.67 bpp 12x12: 0.89 bpp\n\n" - " Supported 3D block sizes are:\n\n" + " Supported 3D block dimensions are:\n\n" " 3x3x3: 4.74 bpp 5x5x4: 1.28 bpp\n" " 4x3x3: 3.56 bpp 5x5x5: 1.02 bpp\n" " 4x4x3: 2.67 bpp 6x5x5: 0.85 bpp\n" @@ -506,7 +506,7 @@ class scApp : public ktxApp { " threads reported by thread::hardware_concurrency or 1 if value\n" " returned is 0.\n" " --verbose Print encoder/compressor activity status to stdout. Currently\n" - " only both astcencoder and Basis Universal compressor emits status.\n" + " only the astc, etc1s and uastc encoders emit status.\n" "\n"; ktxApp::usage(); cerr << endl << @@ -524,7 +524,7 @@ scApp::scApp(string& version, string& defaultVersion, : ktxApp(version, defaultVersion, options), options(options) { argparser::option my_option_list[] = { - { "encode", argparser::option::required_argument, NULL, 'b' }, + { "encode", argparser::option::required_argument, NULL, 'o' }, { "zcmp", argparser::option::optional_argument, NULL, 'z' }, { "no_multithreading", argparser::option::no_argument, NULL, 'N' }, { "threads", argparser::option::required_argument, NULL, 't' }, @@ -547,7 +547,7 @@ scApp::scApp(string& version, string& defaultVersion, { "uastc_rdo_f", argparser::option::no_argument, NULL, 1008 }, { "uastc_rdo_m", argparser::option::no_argument, NULL, 1009 }, { "verbose", argparser::option::no_argument, NULL, 1010 }, - { "astc_blk_s", argparser::option::required_argument, NULL, 1012 }, + { "astc_blk_d", argparser::option::required_argument, NULL, 1012 }, { "astc_func", argparser::option::required_argument, NULL, 1013 }, { "astc_mode", argparser::option::required_argument, NULL, 1014 }, { "astc_quality", argparser::option::required_argument, NULL, 1015 }, @@ -557,7 +557,7 @@ scApp::scApp(string& version, string& defaultVersion, / sizeof(argparser::option); option_list.insert(option_list.begin(), my_option_list, my_option_list + lastOptionIndex); - short_opts += "bz;Nt:c:q:e:E:u:S:n"; + short_opts += "oz;Nt:c:q:e:E:u:S:n"; } void @@ -597,7 +597,7 @@ scApp::processOption(argparser& parser, int opt) bool capture = true; switch (opt) { - case 'b': + case 'o': setEncoder(parser.optarg); options.ktx2 = 1; break; @@ -707,8 +707,8 @@ scApp::processOption(argparser& parser, int opt) options.bopts.noSSE = true; capture = true; break; - case 1012: // astc_blk_s - options.astcopts.blockSize = astcBlockSize(parser.optarg.c_str()); + case 1012: // astc_blk_d + options.astcopts.blockDimension = astcBlockDimension(parser.optarg.c_str()); hasArg = true; break; case 1013: // astc_func From 2bf7bc64f0c47c160ec22c32923f30061bd6599d Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Mon, 24 May 2021 01:56:09 +0100 Subject: [PATCH 05/65] Fix review comments for astc support part 2 --- lib/astc_encode.cpp | 19 -------------- tools/ktxsc/ktxsc.cpp | 8 +++--- tools/toktx/toktx.cc | 26 ++++++++++++++---- utils/scapp.h | 61 +++++++++++++++++-------------------------- 4 files changed, 49 insertions(+), 65 deletions(-) diff --git a/lib/astc_encode.cpp b/lib/astc_encode.cpp index 246e23860c..81c2d6df74 100644 --- a/lib/astc_encode.cpp +++ b/lib/astc_encode.cpp @@ -304,25 +304,6 @@ astcQualityLevel(const char *quality) { return KTX_PACK_ASTC_QUALITY_LEVEL_MEDIUM; } -/** - * @memberof ktxTexture - * @ingroup write - * @~English - * @brief Creates valid astc function from string. - * Checks the input file color space to use as default. - * - * @return Valid ktx_pack_astc_function_e from string - */ -ktx_pack_astc_encoder_function_e -astcEncoderFunction(const char* function) { - if (std::strcmp(function, "srgb") == 0) - return KTX_PACK_ASTC_ENCODER_FUNCTION_SRGB; - else if (std::strcmp(function, "linear") == 0) - return KTX_PACK_ASTC_ENCODER_FUNCTION_LINEAR; - - return KTX_PACK_ASTC_ENCODER_FUNCTION_SRGB; -} - /** * @memberof ktxTexture * @ingroup write diff --git a/tools/ktxsc/ktxsc.cpp b/tools/ktxsc/ktxsc.cpp index 4376bdabf8..07461ad30c 100644 --- a/tools/ktxsc/ktxsc.cpp +++ b/tools/ktxsc/ktxsc.cpp @@ -260,7 +260,7 @@ ktxSupercompressor::main(int argc, _TCHAR *argv[]) exitCode = 1; goto cleanup; } - if ((options.bcmp || options.bopts.uastc) && texture->isCompressed) { + if ((options.etc1s || options.bopts.uastc) && texture->isCompressed) { cerr << name << ": " << "Cannot encode already block-compressed textures " << "to Basis Universal or UASTC." @@ -277,7 +277,7 @@ ktxSupercompressor::main(int argc, _TCHAR *argv[]) (ktx_uint32_t)writer.str().length() + 1, writer.str().c_str()); - if (options.bcmp || options.bopts.uastc) { + if (options.etc1s || options.bopts.uastc) { commandOptions::basisOptions& bopts = options.bopts; ktx_uint32_t transfer = ktxTexture2_GetOETF(texture); if (bopts.normalMap && transfer != KHR_DF_TRANSFER_LINEAR) { @@ -387,8 +387,8 @@ ktxSupercompressor::validateOptions() usage(); exit(1); } - if (!options.bcmp && !options.zcmp && !options.bopts.uastc) { - cerr << "Must specify one of --zcmp, --bcmp or --uastc." << endl; + if (!options.etc1s && !options.zcmp && !options.bopts.uastc) { + cerr << "Must specify one of --zcmp, --etc1s (deprecated --bcmp) or --uastc." << endl; usage(); exit(1); } diff --git a/tools/toktx/toktx.cc b/tools/toktx/toktx.cc index e975c9fee9..7e5efa0b95 100644 --- a/tools/toktx/toktx.cc +++ b/tools/toktx/toktx.cc @@ -724,7 +724,16 @@ toktxApp::main(int argc, _TCHAR *argv[]) image = Image::CreateFromFile(infile, options.assign_oetf == KHR_DF_TRANSFER_UNSPECIFIED, - options.bcmp || options.bopts.uastc || options.astc); + options.etc1s || options.bopts.uastc || + (options.astc && + options.astcopts.mode != KTX_PACK_ASTC_ENCODER_MODE_HDR)); + + if (options.astc && + options.astcopts.mode != KTX_PACK_ASTC_ENCODER_MODE_HDR && + image->getComponentSize() == 2) { + cerr << name << ": Warning! input file is 16bit but no HDR option provided." + << endl; + } if (i == 0) { // First file. @@ -870,7 +879,7 @@ toktxApp::main(int argc, _TCHAR *argv[]) if (options.inputSwizzle.size() > 0 // inputSwizzle is handled during BasisU and astc encoding - && !options.bcmp && !options.bopts.uastc && !options.astc) { + && !options.etc1s && !options.bopts.uastc && !options.astc) { image->swizzle(options.inputSwizzle); } @@ -1106,7 +1115,7 @@ toktxApp::main(int argc, _TCHAR *argv[]) goto cleanup; } #if TRAVIS_DEBUG - if (options.bcmp) { + if (options.etc1s) { cout << "level = " << level << ", faceSlice = " << faceSlice; cout << ", srcImg = " << hex << (void *)srcImg << dec; cout << ", imageSize = " << imageSize << endl; @@ -1201,7 +1210,7 @@ toktxApp::main(int argc, _TCHAR *argv[]) // Add Swizzle metadata if (options.swizzle.size()) { swizzle = options.swizzle; - } else if (!options.bcmp && !options.bopts.uastc && !options.astc + } else if (!options.etc1s && !options.bopts.uastc && !options.astc && defaultSwizzle.size()) { swizzle = defaultSwizzle; } @@ -1229,7 +1238,7 @@ toktxApp::main(int argc, _TCHAR *argv[]) f = _tfopen(options.outfile.c_str(), "wb"); if (f) { - if (options.bcmp || options.bopts.uastc) { + if (options.etc1s || options.bopts.uastc) { commandOptions::basisOptions& bopts = options.bopts; if (bopts.normalMap && chosenOETF != KHR_DF_TRANSFER_LINEAR) { fprintf(stderr, "%s: --normal_map specified but input file(s) are" @@ -1272,6 +1281,13 @@ toktxApp::main(int argc, _TCHAR *argv[]) } } + if (firstImageOETF == KHR_DF_TRANSFER_SRGB) { + astcopts.function = KTX_PACK_ASTC_ENCODER_FUNCTION_SRGB; + } + else { + astcopts.function = KTX_PACK_ASTC_ENCODER_FUNCTION_LINEAR; + } + ret = ktxTexture_CompressAstcEx(texture, &astcopts); if (KTX_SUCCESS != ret) { fprintf(stderr, "%s failed to compress KTX file \"%s\" to astc; KTX error: %s\n", diff --git a/utils/scapp.h b/utils/scapp.h index 3867a84982..200161a5bc 100644 --- a/utils/scapp.h +++ b/utils/scapp.h @@ -44,9 +44,6 @@ astcBlockDimension(const char* block_dimension); ktx_pack_astc_quality_levels_e astcQualityLevel(const char* quality); -ktx_pack_astc_encoder_function_e -astcEncoderFunction(const char* function); - ktx_pack_astc_encoder_mode_e astcEncoderMode(const char* mode); @@ -289,7 +286,7 @@ class scApp : public ktxApp { } }; int ktx2; - int bcmp; + int etc1s; int zcmp; int astc; clamped zcmpLevel; @@ -298,7 +295,7 @@ class scApp : public ktxApp { commandOptions() : zcmpLevel(ZSTD_CLEVEL_DEFAULT, 1U, 22U) { ktx2 = false; - bcmp = false; + etc1s = false; zcmp = false; astc = false; } @@ -324,8 +321,8 @@ class scApp : public ktxApp { void setEncoder(string encoding) { if (encoding == "astc") options.astc = 1; - else if (encoding == "bcmp") - options.bcmp = 1; + else if (encoding == "etc1s") + options.etc1s = 1; else if (encoding == "uastc") options.bopts.uastc = 1; } @@ -333,15 +330,14 @@ class scApp : public ktxApp { void usage() { cerr << - " --encode \n" - " Compress the image data with ETC1S / BasisLZ, high-quality transcodable\n" - " UASTC format or ASTC format. Implies --t2 for all encoding.\n" - " Acceptable options are 'bcmp', 'uastc' and 'astc'. With each encoding\n" - " option the following encoding specific options become valid, otherwise.\n" - " they are ignored.\n\n" + " --encode \n" + " Compress the image data with ASTC format or high-quality transcodable\n" + " ETC1S / BasisLZ or UASTC format. Implies --t2 for all encoding options.\n" + " With each encoding option the following encoder specific options\n" + " become valid, otherwise they are ignored.\n\n" " astc:\n" " Create a texture in high-quality ASTC format.\n" - " --astc_blk_d /\n" + " --astc_blk_d \n" " Specify which block dimension to use for compressing the textures.\n" " e.g. --astc_blk_d 6x5 for 2D or --astc_blk_d 6x6x6 for 3D.\n" " 6x6 is default for 2D.\n\n" @@ -359,13 +355,9 @@ class scApp : public ktxApp { " 4x4x3: 2.67 bpp 6x5x5: 0.85 bpp\n" " 4x4x4: 2.00 bpp 6x6x5: 0.71 bpp\n" " 5x4x4: 1.60 bpp 6x6x6: 0.59 bpp\n" - " --astc_func \n" - " Specify which transfer function to use while compressing.\n" - " Use this only if you don't trust the color space info in source image.\n" - " Default is what's read from the image.\n" - " --astc_mode \n" + " --astc_mode \n" " Specify which encoding mode to use. LDR is the default.\n" - " --astc_quality \n" + " --astc_quality \n" " The quality level configures the quality-performance tradeoff for\n" " the compressor; more complete searches of the search space improve\n" " image quality at the expense of compression time. Default is 'medium'\n" @@ -386,7 +378,7 @@ class scApp : public ktxApp { " nml.xy = texture(...).ga; // Load in [0,1]\n" " nml.xy = nml.xy * 2.0 - 1.0; // Unpack to [-1,1]\n" " nml.z = sqrt(1 - dot(nml.xy, nml.xy)); // Compute Z\n" - " bcmp:\n" + " etc1s:\n" " Supercompress the image data with ETC1S / BasisLZ.\n" " RED images will become RGB with RED in each component. RG images\n" " will have R in the RGB part and G in the alpha part of the\n" @@ -447,7 +439,7 @@ class scApp : public ktxApp { " to do selector RDO.\n\n" " uastc:\n" " Create a texture in high-quality transcodable UASTC format.\n" - " --uastc_level \n" + " --uastc_quality \n" " This optional parameter selects a speed vs quality\n" " tradeoff as shown in the following table:\n" "\n" @@ -495,7 +487,7 @@ class scApp : public ktxApp { " support SSE. Only the Basis Universal compressor uses SSE.\n" " --zcmp []\n" " Supercompress the data with Zstandard. Implies --t2. Can be used\n" - " with data in any format except ETC1S / BasisLZ (bcmp). Most\n" + " with data in any format except ETC1S / BasisLZ. Most\n" " effective with RDO-conditioned UASTC or uncompressed formats. The\n" " optional compressionLevel range is 1 - 22 and the default is 3.\n" " Lower values=faster but give less compression. Values above 20\n" @@ -539,7 +531,7 @@ scApp::scApp(string& version, string& defaultVersion, { "no_endpoint_rdo", argparser::option::no_argument, NULL, 1001 }, { "no_selector_rdo", argparser::option::no_argument, NULL, 1002 }, { "no_sse", argparser::option::no_argument, NULL, 1011 }, - { "uastc_level", argparser::option::required_argument, NULL, 1003 }, + { "uastc_quality", argparser::option::required_argument, NULL, 1003 }, { "uastc_rdo_l", argparser::option::optional_argument, NULL, 1004 }, { "uastc_rdo_d", argparser::option::required_argument, NULL, 1005 }, { "uastc_rdo_b", argparser::option::optional_argument, NULL, 1006 }, @@ -548,10 +540,9 @@ scApp::scApp(string& version, string& defaultVersion, { "uastc_rdo_m", argparser::option::no_argument, NULL, 1009 }, { "verbose", argparser::option::no_argument, NULL, 1010 }, { "astc_blk_d", argparser::option::required_argument, NULL, 1012 }, - { "astc_func", argparser::option::required_argument, NULL, 1013 }, - { "astc_mode", argparser::option::required_argument, NULL, 1014 }, - { "astc_quality", argparser::option::required_argument, NULL, 1015 }, - { "astc_normal", argparser::option::no_argument, NULL, 1016 } + { "astc_mode", argparser::option::required_argument, NULL, 1013 }, + { "astc_quality", argparser::option::required_argument, NULL, 1014 }, + { "astc_normal", argparser::option::no_argument, NULL, 1015 } }; const int lastOptionIndex = sizeof(my_option_list) / sizeof(argparser::option); @@ -602,8 +593,8 @@ scApp::processOption(argparser& parser, int opt) options.ktx2 = 1; break; case 'z': - if (options.bcmp) { - cerr << "Only one of '--encode bcmp' and --zcmp can be specified." + if (options.etc1s) { + cerr << "Only one of '--encode etc1s' and --zcmp can be specified." << endl; usage(); exit(1); @@ -711,19 +702,15 @@ scApp::processOption(argparser& parser, int opt) options.astcopts.blockDimension = astcBlockDimension(parser.optarg.c_str()); hasArg = true; break; - case 1013: // astc_func - options.astcopts.function = astcEncoderFunction(parser.optarg.c_str()); - hasArg = true; - break; - case 1014: // astc_mode + case 1013: // astc_mode options.astcopts.mode = astcEncoderMode(parser.optarg.c_str()); hasArg = true; break; - case 1015: // astc_quality + case 1014: // astc_quality options.astcopts.qualityLevel = astcQualityLevel(parser.optarg.c_str()); hasArg = true; break; - case 1016: // astc_normal + case 1015: // astc_normal options.astcopts.normalMap = true; break; default: From e30fb445e0c9a3d63053281f1ffbf44d59e65983 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Mon, 24 May 2021 02:04:13 +0100 Subject: [PATCH 06/65] Change bcmp command line options to etc1s in tests --- tests/toktx-tests | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/toktx-tests b/tests/toktx-tests index b0dbc15fb8..4aeea2ad6d 100755 --- a/tests/toktx-tests +++ b/tests/toktx-tests @@ -104,18 +104,18 @@ else fi numtests=$numtests+1 -if ! $toktx --zcmp --bcmp a b 2> /dev/null; then +if ! $toktx --zcmp --etc1s a b 2> /dev/null; then passed=$passed+1 else - echo "Simultaneous --bcmp & --zcmp allowed" + echo "Simultaneous --etc1s & --zcmp allowed" failed=$failed+1 fi numtests=$numtests+1 -if ! $toktx --bcmp --uastc a b 2> /dev/null; then +if ! $toktx --etc1s --uastc a b 2> /dev/null; then passed=$passed+1 else - echo "Simultaneous --bcmp & --uastc allowed" + echo "Simultaneous --etc1s & --uastc allowed" failed=$failed+1 fi @@ -197,9 +197,9 @@ gencmpktx rgb-mipmap-reference.ktx --lower_left_maps_to_s0t0 --mipmap --nometada gencmpktx rgb-mipmap-reference-u.ktx2 --test --t2 --mipmap ../srcimages/level0.ppm ../srcimages/level1.ppm ../srcimages/level2.ppm ../srcimages/level3.ppm ../srcimages/level4.ppm ../srcimages/level5.ppm ../srcimages/level6.ppm if [ $(uname) == "Darwin" ]; then # Run only on macOS until we figure out the Basis compressor non-determinancy. - gencmpktx alpha_simple_basis.ktx2 --test --encode bcmp ../srcimages/alpha_simple.png - gencmpktx kodim17_basis.ktx2 --test --encode bcmp ../srcimages/kodim17.png - gencmpktx color_grid_basis.ktx2 --test --encode bcmp ../srcimages/color_grid.png + gencmpktx alpha_simple_basis.ktx2 --test --encode etc1s ../srcimages/alpha_simple.png + gencmpktx kodim17_basis.ktx2 --test --encode etc1s ../srcimages/kodim17.png + gencmpktx color_grid_basis.ktx2 --test --encode etc1s ../srcimages/color_grid.png gencmpktx cimg5293_uastc.ktx2 --encode uastc --genmipmap --test ../srcimages/CIMG5293.jpg gencmpktx cimg5293_uastc_zstd.ktx2 --zcmp --encode uastc --genmipmap --test ../srcimages/CIMG5293.jpg fi From a806477d4918406d3b21f35fefa94e5c40f6a67f Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Mon, 24 May 2021 10:13:26 +0100 Subject: [PATCH 07/65] Add back deprecated options --bcmp and --uastc --- tests/toktx-tests | 19 ++++---------- utils/scapp.h | 63 ++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 59 insertions(+), 23 deletions(-) diff --git a/tests/toktx-tests b/tests/toktx-tests index 4aeea2ad6d..1202a7c9e5 100755 --- a/tests/toktx-tests +++ b/tests/toktx-tests @@ -164,18 +164,9 @@ function gencmpktx() { local args local reference=$1; shift local tempfile=$(mktmp) - local prevarg="" for i in $*; do - if [ ! -z "$prevarg" ]; then - args="$args $i" - prevarg="" - shift - fi if [ ${i:0:2} == "--" ]; then args="$args $i" - if [ ${i:2:6} == "encode" ]; then - prevarg="encode" - fi shift fi done @@ -197,11 +188,11 @@ gencmpktx rgb-mipmap-reference.ktx --lower_left_maps_to_s0t0 --mipmap --nometada gencmpktx rgb-mipmap-reference-u.ktx2 --test --t2 --mipmap ../srcimages/level0.ppm ../srcimages/level1.ppm ../srcimages/level2.ppm ../srcimages/level3.ppm ../srcimages/level4.ppm ../srcimages/level5.ppm ../srcimages/level6.ppm if [ $(uname) == "Darwin" ]; then # Run only on macOS until we figure out the Basis compressor non-determinancy. - gencmpktx alpha_simple_basis.ktx2 --test --encode etc1s ../srcimages/alpha_simple.png - gencmpktx kodim17_basis.ktx2 --test --encode etc1s ../srcimages/kodim17.png - gencmpktx color_grid_basis.ktx2 --test --encode etc1s ../srcimages/color_grid.png - gencmpktx cimg5293_uastc.ktx2 --encode uastc --genmipmap --test ../srcimages/CIMG5293.jpg - gencmpktx cimg5293_uastc_zstd.ktx2 --zcmp --encode uastc --genmipmap --test ../srcimages/CIMG5293.jpg + gencmpktx alpha_simple_basis.ktx2 --test --bcmp ../srcimages/alpha_simple.png + gencmpktx kodim17_basis.ktx2 --test --bcmp ../srcimages/kodim17.png + gencmpktx color_grid_basis.ktx2 --test --bcmp ../srcimages/color_grid.png + gencmpktx cimg5293_uastc.ktx2 --uastc --genmipmap --test ../srcimages/CIMG5293.jpg + gencmpktx cimg5293_uastc_zstd.ktx2 --zcmp --uastc --genmipmap --test ../srcimages/CIMG5293.jpg fi filelist=$(mktemp) diff --git a/utils/scapp.h b/utils/scapp.h index 200161a5bc..e607d844ec 100644 --- a/utils/scapp.h +++ b/utils/scapp.h @@ -363,11 +363,13 @@ class scApp : public ktxApp { " image quality at the expense of compression time. Default is 'medium'\n" " The quality level can be set to fastest (0) and thorough (100) via the \n" " following fixed quality presets:\n\n" - " fastest (equivalent to quality = 0)\n" - " fast (equivalent to quality = 10)\n" - " medium (equivalent to quality = 60)\n" - " thorough (equivalent to quality = 98)\n" - " exhaustive (equivalent to quality = 100)\n" + " Level | Quality\n" + " ---------- | -----------------------------\n" + " fastest | (equivalent to quality = 0)\n" + " fast | (equivalent to quality = 10)\n" + " medium | (equivalent to quality = 60)\n" + " thorough | (equivalent to quality = 98)\n" + " exhaustive | (equivalent to quality = 100)\n" " --astc_normal\n" " The input texture is a three component linear LDR normal map\n" " storing unit length normals as (R=X, G=Y, B=Z). The output will\n" @@ -444,7 +446,7 @@ class scApp : public ktxApp { " tradeoff as shown in the following table:\n" "\n" " Level | Speed | Quality\n" - " ----- | ------- | -------\n" + " ----- | --------- | -------\n" " 0 | Fastest | 43.45dB\n" " 1 | Faster | 46.49dB\n" " 2 | Default | 47.47dB\n" @@ -485,6 +487,10 @@ class scApp : public ktxApp { " deterministic).\n\n" " --no_sse Forbid use of the SSE instruction set. Ignored if CPU does not\n" " support SSE. Only the Basis Universal compressor uses SSE.\n" + " --bcmp\n" + " Deprecated. Use '--encode etc1s' instead.\n" + " --uastc []\n" + " Deprecated. Use '--encode uastc' instead.\n" " --zcmp []\n" " Supercompress the data with Zstandard. Implies --t2. Can be used\n" " with data in any format except ETC1S / BasisLZ. Most\n" @@ -542,13 +548,16 @@ scApp::scApp(string& version, string& defaultVersion, { "astc_blk_d", argparser::option::required_argument, NULL, 1012 }, { "astc_mode", argparser::option::required_argument, NULL, 1013 }, { "astc_quality", argparser::option::required_argument, NULL, 1014 }, - { "astc_normal", argparser::option::no_argument, NULL, 1015 } + { "astc_normal", argparser::option::no_argument, NULL, 1015 }, + // Deprecated options + { "bcmp", argparser::option::no_argument, NULL, 'b' }, + { "uastc", argparser::option::optional_argument, NULL, 1016 } }; const int lastOptionIndex = sizeof(my_option_list) / sizeof(argparser::option); option_list.insert(option_list.begin(), my_option_list, my_option_list + lastOptionIndex); - short_opts += "oz;Nt:c:q:e:E:u:S:n"; + short_opts += "oz;Nt:c:q:e:E:u:S:nb"; } void @@ -594,7 +603,7 @@ scApp::processOption(argparser& parser, int opt) break; case 'z': if (options.etc1s) { - cerr << "Only one of '--encode etc1s' and --zcmp can be specified." + cerr << "Only one of '--encode etc1s|--bcmp' and --zcmp can be specified." << endl; usage(); exit(1); @@ -713,6 +722,42 @@ scApp::processOption(argparser& parser, int opt) case 1015: // astc_normal options.astcopts.normalMap = true; break; + case 'b': + if (options.zcmp) { + cerr << "Only one of --bcmp and --zcmp can be specified.\n" + << "--bcmp is deprecated, use '--encode etc1s' instead." + << endl; + usage(); + exit(1); + } + if (options.bopts.uastc) { + cerr << "Only one of --bcmp and '--encode etc1s|--uastc' can be specified.\n" + << "--bcmp is deprecated, use '--encode etc1s' instead." + << endl; + usage(); + exit(1); + } + options.etc1s = 1; + options.ktx2 = 1; + break; + case 1016: + if (options.etc1s) { + cerr << "Only one of `--encode etc1s|--bcmp` and `--uastc []` can be specified." + << endl; + usage(); + exit(1); + } + options.bopts.uastc = 1; + options.ktx2 = 1; + if (parser.optarg.size() > 0) { + ktx_uint32_t level = strtoi(parser.optarg.c_str()); + level = clamp(level, 0, KTX_PACK_UASTC_MAX_LEVEL); + // Ensure the last one wins in case of multiple of these args. + options.bopts.uastcFlags = (unsigned int)~KTX_PACK_UASTC_LEVEL_MASK; + options.bopts.uastcFlags |= level; + hasArg = true; + } + break; default: return false; } From d56a6e9465b0875bdc9f518ca269d2e5b1f2d401 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Mon, 24 May 2021 12:53:57 +0100 Subject: [PATCH 08/65] Update patch up code in ktxTexture_CompressAstcEx to use a prototype texture --- lib/astc_encode.cpp | 139 ++++++++++++++++++++++++++++++-------------- 1 file changed, 94 insertions(+), 45 deletions(-) diff --git a/lib/astc_encode.cpp b/lib/astc_encode.cpp index 81c2d6df74..2a23c58b72 100644 --- a/lib/astc_encode.cpp +++ b/lib/astc_encode.cpp @@ -11,7 +11,7 @@ * @file astc_encode.cpp * @~English * - * @brief Functions for compressing a texture to astc format. + * @brief Functions for compressing a texture to ASTC format. * * @author Wasim Abbas , www.arm.com */ @@ -213,9 +213,9 @@ astcBufferSize(uint32_t width, uint32_t height, uint32_t depth, * @memberof ktxTexture * @ingroup write * @~English - * @brief Creates default astc parameters + * @brief Creates default ASTC parameters * - * @return ktxAstcParams with default options for astc compressor + * @return ktxAstcParams with default options for ASTC compressor */ static ktxAstcParams astcDefaultOptions() { @@ -236,7 +236,7 @@ astcDefaultOptions() { * @memberof ktxTexture * @ingroup write * @~English - * @brief Creates valid astc block dimension from string. + * @brief Creates valid ASTC block dimension from string. * * @return Valid ktx_pack_astc_block_dimension_e from string */ @@ -280,7 +280,7 @@ astcBlockDimension(const char* block_size) { * @memberof ktxTexture * @ingroup write * @~English - * @brief Creates valid astc quality from string. + * @brief Creates valid ASTC quality from string. * * @return Valid ktx_pack_astc_quality_e from string */ @@ -308,7 +308,7 @@ astcQualityLevel(const char *quality) { * @memberof ktxTexture * @ingroup write * @~English - * @brief Creates valid astc mode from string. + * @brief Creates valid ASTC mode from string. * * @return Valid ktx_pack_astc_mode_e from string */ @@ -326,11 +326,11 @@ astcEncoderMode(const char* mode) { * @memberof ktxTexture * @ingroup write * @~English - * @brief Should be used to get VkFormat from astc block enum + * @brief Should be used to get VkFormat from ASTC block enum * - * @return VKFormat for a specific astc block size + * @return VKFormat for a specific ASTC block size */ -static ktx_uint32_t +static VkFormat astcVkFormat(ktx_uint32_t block_size, bool sRGB) { if (sRGB) { switch (block_size) { @@ -395,7 +395,7 @@ astcVkFormat(ktx_uint32_t block_size, bool sRGB) { * @memberof ktxTexture * @ingroup write * @~English - * @brief Creates valid astc encoder action from string. + * @brief Creates valid ASTC encoder action from string. * * @return Valid astc_profile from string */ @@ -440,7 +440,7 @@ astcEncoderAction(const ktxAstcParams ¶ms, const uint32_t* bdb) { * @memberof ktxTexture * @ingroup write * @~English - * @brief Creates valid astc encoder swizzle from string. + * @brief Creates valid ASTC encoder swizzle from string. * * @return Valid astcenc_swizzle from string */ @@ -606,14 +606,14 @@ launchThreads(int threadCount, void (*func)(int, int, void*), void *payload) { * @~English * @brief Encode and compress a ktx texture with uncompressed images to astc. * - * The images are encoded to astc block-compressed format. The encoded images + * The images are encoded to ASTC block-compressed format. The encoded images * replace the original images and the texture's fields including the DFD are * modified to reflect the new state. * * Such textures can be directly uploaded to a GPU via a graphics API. * * @param[in] This pointer to the ktxTexture object of interest. - * @param[in] params pointer to astc params object. + * @param[in] params pointer to ASTC params object. * * @return KTX_SUCCESS on success, other KTX_* enum values on error. * @@ -632,14 +632,14 @@ launchThreads(int threadCount, void (*func)(int, int, void*), void *payload) { * The texture's images are 1D. Only 2D images can * be supercompressed. * @exception KTX_INVALID_OPERATION - * Astc compressor failed to compress image for any + * ASTC compressor failed to compress image for any reason. * @exception KTX_OUT_OF_MEMORY Not enough memory to carry out compression. */ extern "C" KTX_error_code ktxTexture_CompressAstcEx(ktxTexture* _This, ktxAstcParams* params) { // FIXME: At the moment defaults to ktx2 textures only - assert(_This->classId == ktxTexture2_c && "Only support ktx2 astc."); + assert(_This->classId == ktxTexture2_c && "Only support ktx2 ASTC."); ktxTexture2* This = (ktxTexture2*)_This; KTX_error_code result; @@ -655,7 +655,7 @@ ktxTexture_CompressAstcEx(ktxTexture* _This, ktxAstcParams* params) { if (This->isCompressed) return KTX_INVALID_OPERATION; // Only non-block compressed formats - // can be encoded into a astc format. + // can be encoded into an ASTC format. if (This->_protected->_formatSize.flags & KTX_FORMAT_SIZE_PACKED_BIT) return KTX_INVALID_OPERATION; @@ -680,6 +680,37 @@ ktxTexture_CompressAstcEx(ktxTexture* _This, ktxAstcParams* params) { if (threadCount < 1) threadCount = 1; + ktx_uint32_t transfer = KHR_DFDVAL(BDB, TRANSFER); + bool sRGB = transfer == KHR_DF_TRANSFER_SRGB; + + VkFormat vkFormat = astcVkFormat(params->blockDimension, sRGB); + + // Create a prototype texture to use for calculating sizes in the target + // format and, as useful side effects, provide us with a properly sized + // data allocation and the DFD for the target format. + ktxTextureCreateInfo createInfo; + createInfo.glInternalformat = 0; + createInfo.vkFormat = vkFormat; + createInfo.baseWidth = This->baseWidth; + createInfo.baseHeight = This->baseHeight; + createInfo.baseDepth = This->baseDepth; + createInfo.generateMipmaps = This->generateMipmaps; + createInfo.isArray = This->isArray; + createInfo.numDimensions = This->numDimensions; + createInfo.numFaces = This->numFaces; + createInfo.numLayers = This->numLayers; + createInfo.numLevels = This->numLevels; + createInfo.pDfd = nullptr; + + ktxTexture2* prototype; + result = ktxTexture2_Create(&createInfo, KTX_TEXTURE_CREATE_ALLOC_STORAGE, + &prototype); + + if (result != KTX_SUCCESS) { + assert(result == KTX_OUT_OF_MEMORY && "Out of memory allocating texture."); + return result; + } + astcenc_profile profile{ASTCENC_PRF_LDR_SRGB}; astcenc_swizzle swizzle{ASTCENC_SWZ_R, ASTCENC_SWZ_G, ASTCENC_SWZ_B, ASTCENC_SWZ_A}; @@ -704,7 +735,7 @@ ktxTexture_CompressAstcEx(ktxTexture* _This, ktxAstcParams* params) { &astc_config); if (astc_error != ASTCENC_SUCCESS) { - std::cout << "Astc config init failed\n"; + std::cout << "ASTC config init failed\n"; return KTX_INVALID_OPERATION; } @@ -712,7 +743,7 @@ ktxTexture_CompressAstcEx(ktxTexture* _This, ktxAstcParams* params) { &astc_context); if (astc_error != ASTCENC_SUCCESS) { - std::cout << "Astc context alloc failed\n"; + std::cout << "ASTC context alloc failed\n"; return KTX_INVALID_OPERATION; } @@ -729,10 +760,10 @@ ktxTexture_CompressAstcEx(ktxTexture* _This, ktxAstcParams* params) { astc_input_images.reserve(num_images); astc_image_sizes.reserve(num_images); - // Size of all astc compressed images in bytes + // Size of all ASTC compressed images in bytes ktx_size_t astc_images_size = 0; - ktxTexture2_private& priv = *This->_private; + ktxTexture2_private& priv = *prototype->_private; ktx_size_t level_offset = 0; @@ -787,9 +818,10 @@ ktxTexture_CompressAstcEx(ktxTexture* _This, ktxAstcParams* params) { } } - // Offset is from start of the astc compressed block + + // Offset is from start of the ASTC compressed block priv._levelIndex[level].byteOffset = level_offset; - priv._requiredLevelAlignment = 8; // For astc its always 8 bytes (128bits) irrespective of block sizes + priv._requiredLevelAlignment = 8; // For ASTC its always 8 bytes (128bits) irrespective of block sizes ktx_size_t prev_level_offset = level_offset; level_offset += _KTX_PADN(priv._requiredLevelAlignment, @@ -806,18 +838,18 @@ ktxTexture_CompressAstcEx(ktxTexture* _This, ktxAstcParams* params) { This->dataSize = 0; // Allocate big enough buffer for all compressed images - This->dataSize = astc_images_size; - This->pData = reinterpret_cast(malloc(This->dataSize)); - if (!This->pData) { + prototype->dataSize = astc_images_size; + prototype->pData = reinterpret_cast(malloc(prototype->dataSize)); + if (!prototype->pData) { return KTX_OUT_OF_MEMORY; } - uint8_t* buffer_out = This->pData; + uint8_t* buffer_out = prototype->pData; uint32_t input_size = astc_input_images.size(); for (uint32_t level = 0; level < input_size; level++) { if (params->verbose) - std::cout << "Astc compressor: compressing image = " << + std::cout << "ASTC compressor: compressing image = " << level + 1 << " of " << input_size << std::endl; // Lets compress to astc @@ -835,7 +867,7 @@ ktxTexture_CompressAstcEx(ktxTexture* _This, ktxAstcParams* params) { launchThreads(threadCount, compressionWorkloadRunner, &work); if (work.error != ASTCENC_SUCCESS) { - std::cout << "Astc compressor failed\n" << + std::cout << "ASTC compressor failed\n" << astcenc_get_error_string(work.error) << std::endl; for(auto& ii : astc_input_images) @@ -850,37 +882,54 @@ ktxTexture_CompressAstcEx(ktxTexture* _This, ktxAstcParams* params) { // Free input image imageFree(input_image); - // Reset astc context for next image + // Reset ASTC context for next image astcenc_compress_reset(astc_context); } // We are done with astcencoder astcenc_context_free(astc_context); - ktx_uint32_t transfer = KHR_DFDVAL(BDB, TRANSFER); - bool sRGB = transfer == KHR_DF_TRANSFER_SRGB; // Is this right or I need "KHR_DF_PRIMARIES_SRGB"? - - This->vkFormat = astcVkFormat(params->blockDimension, sRGB); - - free(This->pDfd); - This->pDfd = vk2dfd(static_cast(This->vkFormat)); - // reset pointer to new BDB - BDB = This->pDfd+1; + BDB = prototype->pDfd+1; - // Astc-related checks + // ASTC-related checks ktx_uint32_t model = KHR_DFDVAL(BDB, MODEL); ktx_uint32_t primaries = KHR_DFDVAL(BDB, PRIMARIES); - This->isCompressed = true; - assert(model == KHR_DF_MODEL_ASTC && "Invalid dfd generated for astc image\n"); + assert(model == KHR_DF_MODEL_ASTC && "Invalid dfd generated for ASTC image\n"); if (transfer == KHR_DF_TRANSFER_SRGB) { assert(primaries == KHR_DF_PRIMARIES_SRGB && "Not a valid sRGB image\n"); } - // Block-compressed textures never need byte swapping so typeSize is 1. - This->_protected->_typeSize = 1; - + // Fix up the current (This) texture +#undef DECLARE_PRIVATE +#undef DECLARE_PROTECTED +#define DECLARE_PRIVATE(n,t2) ktxTexture2_private& n = *(t2->_private) +#define DECLARE_PROTECTED(n,t2) ktxTexture_protected& n = *(t2->_protected) + + DECLARE_PROTECTED(thisPrtctd, This); + DECLARE_PRIVATE(protoPriv, prototype); + DECLARE_PROTECTED(protoPrtctd, prototype); + memcpy(&thisPrtctd._formatSize, &protoPrtctd._formatSize, + sizeof(ktxFormatSize)); + This->vkFormat = vkFormat; + This->isCompressed = prototype->isCompressed; + This->supercompressionScheme = KTX_SS_NONE; + This->_private->_requiredLevelAlignment = protoPriv._requiredLevelAlignment; + // Copy the levelIndex from the prototype to This. + memcpy(This->_private->_levelIndex, protoPriv._levelIndex, + This->numLevels * sizeof(ktxLevelIndexEntry)); + // Move the DFD and data from the prototype to This. + delete This->pDfd; + This->pDfd = prototype->pDfd; + prototype->pDfd = 0; + delete This->pData; + This->pData = prototype->pData; + This->dataSize = prototype->dataSize; + prototype->pData = 0; + prototype->dataSize = 0; + + ktxTexture2_Destroy(prototype); return KTX_SUCCESS; } @@ -890,7 +939,7 @@ ktxTexture_CompressAstcEx(ktxTexture* _This, ktxAstcParams* params) { * @~English * @brief Encode and compress a ktx texture with uncompressed images to astc. * - * The images are either encoded to astc block-compressed format. The encoded images + * The images are either encoded to ASTC block-compressed format. The encoded images * replace the original images and the texture's fields including the DFD are modified to reflect the new * state. * From 99e0a72d341aac91f0546b4e41166c19ee6d44d0 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Mon, 24 May 2021 13:05:16 +0100 Subject: [PATCH 09/65] Remove further redundant code, Since we are now using prototype texture no more levelIndex calculation is required --- lib/astc_encode.cpp | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/lib/astc_encode.cpp b/lib/astc_encode.cpp index 2a23c58b72..52d72dc627 100644 --- a/lib/astc_encode.cpp +++ b/lib/astc_encode.cpp @@ -763,10 +763,6 @@ ktxTexture_CompressAstcEx(ktxTexture* _This, ktxAstcParams* params) { // Size of all ASTC compressed images in bytes ktx_size_t astc_images_size = 0; - ktxTexture2_private& priv = *prototype->_private; - - ktx_size_t level_offset = 0; - // Walk in reverse on levels so we don't have to do this later // This->numLevels = 0 not allowed for block compressed formats // But just in case make sure its not zero @@ -778,9 +774,6 @@ ktxTexture_CompressAstcEx(ktxTexture* _This, ktxAstcParams* params) { uint32_t depth = MAX(1, This->baseDepth >> level); uint32_t faceSlices = This->numFaces == 1 ? depth : This->numFaces; - priv._levelIndex[level].byteLength = 0; - priv._levelIndex[level].uncompressedByteLength = 0; - ktx_size_t size = astcBufferSize(width, height, depth, block_size_x, block_size_y, block_size_z); @@ -812,25 +805,8 @@ ktxTexture_CompressAstcEx(ktxTexture* _This, ktxAstcParams* params) { astc_image_sizes.push_back(size); astc_images_size += size; - - priv._levelIndex[level].byteLength += size; - priv._levelIndex[level].uncompressedByteLength += size; } } - - - // Offset is from start of the ASTC compressed block - priv._levelIndex[level].byteOffset = level_offset; - priv._requiredLevelAlignment = 8; // For ASTC its always 8 bytes (128bits) irrespective of block sizes - - ktx_size_t prev_level_offset = level_offset; - level_offset += _KTX_PADN(priv._requiredLevelAlignment, - priv._levelIndex[level].byteLength); - - ktx_size_t diff = level_offset - prev_level_offset; - - assert(diff - priv._levelIndex[level].byteLength == 0 - && "\nSome format/blocks aren't alighned to 8 bytes, need to create offsets\n"); } free(This->pData); // No longer needed. Reduce memory footprint. From bf19acde14ef7a5c91c2767ea18c6aa9c301c6f5 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Mon, 24 May 2021 15:35:14 +0100 Subject: [PATCH 10/65] Remove redundant loop over the levels of the image --- lib/astc_encode.cpp | 178 ++++++++++++++++---------------------------- 1 file changed, 66 insertions(+), 112 deletions(-) diff --git a/lib/astc_encode.cpp b/lib/astc_encode.cpp index 52d72dc627..a0b581495a 100644 --- a/lib/astc_encode.cpp +++ b/lib/astc_encode.cpp @@ -199,16 +199,6 @@ unorm8x4ArrayToImage(const uint8_t *data, uint32_t dim_x, uint32_t dim_y) { return img; } -static ktx_size_t -astcBufferSize(uint32_t width, uint32_t height, uint32_t depth, - uint32_t block_x, uint32_t block_y, uint32_t block_z) { - auto xblocs = (width + block_x - 1) / block_x; - auto yblocs = (height + block_y - 1) / block_y; - auto zblocs = (depth + block_z - 1) / block_z; - - return xblocs * yblocs * zblocs * 16; -} - /** * @memberof ktxTexture * @ingroup write @@ -747,119 +737,83 @@ ktxTexture_CompressAstcEx(ktxTexture* _This, ktxAstcParams* params) { return KTX_INVALID_OPERATION; } - // - // Copy images into compressor readable format. - // - std::vector astc_input_images; - std::vector astc_image_sizes; - - // Conservative allocation, only used to resize data vectors - uint32_t num_images = MAX(1, This->numLayers) * MAX(1, This->numFaces) * - MAX(1, This->numLevels) * MAX(1, This->baseDepth); - - astc_input_images.reserve(num_images); - astc_image_sizes.reserve(num_images); - - // Size of all ASTC compressed images in bytes - ktx_size_t astc_images_size = 0; - // Walk in reverse on levels so we don't have to do this later // This->numLevels = 0 not allowed for block compressed formats // But just in case make sure its not zero This->numLevels = MAX(1, This->numLevels); - for (int32_t level = This->numLevels - 1; level >= 0; level--) { - uint32_t width = MAX(1, This->baseWidth >> level); - uint32_t height = MAX(1, This->baseHeight >> level); - uint32_t depth = MAX(1, This->baseDepth >> level); - uint32_t faceSlices = This->numFaces == 1 ? depth : This->numFaces; - - ktx_size_t size = astcBufferSize(width, height, depth, - block_size_x, block_size_y, block_size_z); - - for (uint32_t layer = 0; layer < MAX(1, This->numLayers); layer++) { - for (ktx_uint32_t slice = 0; slice < faceSlices; slice++) { - ktx_size_t offset; - ktxTexture2_GetImageOffset((ktxTexture2*)This, level, layer, - slice, &offset); - - // TODO: Fix depth for 3D textures - // Create compressor readable image from each image in the container - astcenc_image *input_image = nullptr; - if (num_components == 1) - input_image = unorm8x1ArrayToImage(This->pData + offset, - width, height); - else if (num_components == 2) - input_image = unorm8x2ArrayToImage(This->pData + offset, - width, height); - else if (num_components == 3) - input_image = unorm8x3ArrayToImage(This->pData + offset, - width, height); - else // assume (num_components == 4) - input_image = unorm8x4ArrayToImage(This->pData + offset, - width, height); - - assert(input_image); - - astc_input_images.push_back(input_image); - astc_image_sizes.push_back(size); - - astc_images_size += size; - } - } - } - - free(This->pData); // No longer needed. Reduce memory footprint. - This->pData = NULL; - This->dataSize = 0; + assert(prototype->dataSize && "Prototype texture size not initialized.\n"); - // Allocate big enough buffer for all compressed images - prototype->dataSize = astc_images_size; - prototype->pData = reinterpret_cast(malloc(prototype->dataSize)); if (!prototype->pData) { return KTX_OUT_OF_MEMORY; } uint8_t* buffer_out = prototype->pData; - uint32_t input_size = astc_input_images.size(); - - for (uint32_t level = 0; level < input_size; level++) { - if (params->verbose) - std::cout << "ASTC compressor: compressing image = " << - level + 1 << " of " << input_size << std::endl; - - // Lets compress to astc - astcenc_image *input_image = astc_input_images[level]; - ktx_size_t size = astc_image_sizes[level]; - - CompressionWorkload work; - work.context = astc_context; - work.image = input_image; - work.swizzle = swizzle; - work.data_out = buffer_out; - work.data_len = size; - work.error = ASTCENC_SUCCESS; - launchThreads(threadCount, compressionWorkloadRunner, &work); - - if (work.error != ASTCENC_SUCCESS) { - std::cout << "ASTC compressor failed\n" << - astcenc_get_error_string(work.error) << std::endl; + for (int32_t level = This->numLevels - 1; level >= 0; level--) { + uint32_t width = MAX(1, This->baseWidth >> level); + uint32_t height = MAX(1, This->baseHeight >> level); + uint32_t depth = MAX(1, This->baseDepth >> level); + ktx_size_t levelImageSizeIn = 0; + ktx_size_t levelImageSizeOut = 0; + ktx_uint32_t levelImages = 0; + + levelImages = This->numLayers * This->numFaces * depth; + levelImageSizeIn = ktxTexture_calcImageSize(ktxTexture(This), level, + KTX_FORMAT_VERSION_TWO); + levelImageSizeOut = ktxTexture_calcImageSize(ktxTexture(prototype), level, + KTX_FORMAT_VERSION_TWO); + ktx_size_t offset = ktxTexture2_levelDataOffset(This, level); + + for (uint32_t image = 0; image < levelImages; image++) { + if (params->verbose) + std::cout << "ASTC compressor: compressing image " << + (This->numLevels - level - 1) * levelImages + image + 1 + << " of " << This->numLevels * levelImages + << std::endl; + + astcenc_image *input_image = nullptr; + if (num_components == 1) + input_image = unorm8x1ArrayToImage(This->pData + offset, + width, height); + else if (num_components == 2) + input_image = unorm8x2ArrayToImage(This->pData + offset, + width, height); + else if (num_components == 3) + input_image = unorm8x3ArrayToImage(This->pData + offset, + width, height); + else // assume (num_components == 4) + input_image = unorm8x4ArrayToImage(This->pData + offset, + width, height); + + assert(input_image); + + CompressionWorkload work; + work.context = astc_context; + work.image = input_image; + work.swizzle = swizzle; + work.data_out = buffer_out; + work.data_len = levelImageSizeOut; + work.error = ASTCENC_SUCCESS; + + launchThreads(threadCount, compressionWorkloadRunner, &work); + + if (work.error != ASTCENC_SUCCESS) { + std::cout << "ASTC compressor failed\n" << + astcenc_get_error_string(work.error) << std::endl; + + imageFree(input_image); + + astcenc_context_free(astc_context); + return KTX_INVALID_OPERATION; + } - for(auto& ii : astc_input_images) - imageFree(ii); + buffer_out += levelImageSizeOut; - astcenc_context_free(astc_context); - return KTX_INVALID_OPERATION; + // Reset ASTC context for next image + astcenc_compress_reset(astc_context); + offset += levelImageSizeIn; } - - buffer_out += size; - - // Free input image - imageFree(input_image); - - // Reset ASTC context for next image - astcenc_compress_reset(astc_context); } // We are done with astcencoder @@ -878,10 +832,10 @@ ktxTexture_CompressAstcEx(ktxTexture* _This, ktxAstcParams* params) { } // Fix up the current (This) texture -#undef DECLARE_PRIVATE -#undef DECLARE_PROTECTED -#define DECLARE_PRIVATE(n,t2) ktxTexture2_private& n = *(t2->_private) -#define DECLARE_PROTECTED(n,t2) ktxTexture_protected& n = *(t2->_protected) + #undef DECLARE_PRIVATE + #undef DECLARE_PROTECTED + #define DECLARE_PRIVATE(n,t2) ktxTexture2_private& n = *(t2->_private) + #define DECLARE_PROTECTED(n,t2) ktxTexture_protected& n = *(t2->_protected) DECLARE_PROTECTED(thisPrtctd, This); DECLARE_PRIVATE(protoPriv, prototype); From 0ac3ab12f915b20380a8e8b3460e88c0fd020815 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Mon, 24 May 2021 16:26:16 +0100 Subject: [PATCH 11/65] Fix CI by adding ISA_NONE to astcenc options --- CMakeLists.txt | 3 +++ ci_scripts/build_android.sh | 3 ++- ci_scripts/build_android_debug.sh | 3 ++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4a4b3e380d..7e82d5e8a0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -648,6 +648,9 @@ elseif(CPU_ARCHITECTURE STREQUAL arm OR CPU_ARCHITECTURE STREQUAL armv7 OR CPU_ARCHITECTURE STREQUAL armv8 OR CPU_ARCHITECTURE STREQUAL arm64) set(ISA_NEON ON) set(ASTC_LIB_NAME astcenc-neon-static) +else() + set(ISA_NONE ON) + set(ASTC_LIB_NAME astcenc-none-static) endif() # astcenc diff --git a/ci_scripts/build_android.sh b/ci_scripts/build_android.sh index 42408cb544..0ae79282f1 100755 --- a/ci_scripts/build_android.sh +++ b/ci_scripts/build_android.sh @@ -17,7 +17,8 @@ cmake . -G Ninja -B "build-android-$ANDROID_ABI" \ -DANDROID_NDK="$ANDROID_NDK" \ -DCMAKE_TOOLCHAIN_FILE="$ANDROID_NDK/build/cmake/android.toolchain.cmake" \ -DCMAKE_BUILD_TYPE=Release \ --DBASISU_SUPPORT_SSE=OFF +-DBASISU_SUPPORT_SSE=OFF \ +-DISA_NONE=ON pushd "build-android-$ANDROID_ABI" diff --git a/ci_scripts/build_android_debug.sh b/ci_scripts/build_android_debug.sh index dfa79e8fc6..66698c1fc7 100755 --- a/ci_scripts/build_android_debug.sh +++ b/ci_scripts/build_android_debug.sh @@ -17,7 +17,8 @@ cmake . -G Ninja -B "build-android-$ANDROID_ABI-debug" \ -DANDROID_NDK="$ANDROID_NDK" \ -DCMAKE_TOOLCHAIN_FILE="$ANDROID_NDK/build/cmake/android.toolchain.cmake" \ -DCMAKE_BUILD_TYPE=Debug \ --DBASISU_SUPPORT_SSE=OFF +-DBASISU_SUPPORT_SSE=OFF \ +-DISA_NONE=ON pushd "build-android-$ANDROID_ABI-debug" From ecee952aba1fafc9cc09547c757553d187e85278 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Mon, 24 May 2021 16:42:43 +0100 Subject: [PATCH 12/65] Fix linux ci build --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7e82d5e8a0..a9637868ec 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -655,6 +655,7 @@ endif() # astcenc add_subdirectory(lib/astc-encoder) +set_property(TARGET ${ASTC_LIB_NAME} PROPERTY POSITION_INDEPENDENT_CODE ON) target_link_libraries(ktx PRIVATE ${ASTC_LIB_NAME}) # Tools From 18af4983456764eb784babe2d11a125f5d2b7f5c Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Tue, 25 May 2021 19:15:14 +0100 Subject: [PATCH 13/65] Fix review comments Add common '--normal_mode', common 'threadCount' option, replaced delete with free --- lib/astc_encode.cpp | 22 ++++++-------- lib/basis_encode.cpp | 2 +- lib/basis_transcode.cpp | 4 +-- tools/toktx/toktx.cc | 17 +++++++---- utils/scapp.h | 63 +++++++++++++++++++++-------------------- 5 files changed, 55 insertions(+), 53 deletions(-) diff --git a/lib/astc_encode.cpp b/lib/astc_encode.cpp index a0b581495a..09eaf8e7d0 100644 --- a/lib/astc_encode.cpp +++ b/lib/astc_encode.cpp @@ -712,7 +712,7 @@ ktxTexture_CompressAstcEx(ktxTexture* _This, ktxAstcParams* params) { uint32_t flags{params->normalMap ? ASTCENC_FLG_MAP_NORMAL : 0}; astcBlockDimensions(params->blockDimension, - block_size_x, block_size_y, block_size_z); + block_size_x, block_size_y, block_size_z); quality = astcQuality(params->qualityLevel); profile = astcEncoderAction(*params, BDB); swizzle = astcSwizzle(*params); @@ -819,17 +819,11 @@ ktxTexture_CompressAstcEx(ktxTexture* _This, ktxAstcParams* params) { // We are done with astcencoder astcenc_context_free(astc_context); - // reset pointer to new BDB - BDB = prototype->pDfd+1; - - // ASTC-related checks - ktx_uint32_t model = KHR_DFDVAL(BDB, MODEL); - ktx_uint32_t primaries = KHR_DFDVAL(BDB, PRIMARIES); - - assert(model == KHR_DF_MODEL_ASTC && "Invalid dfd generated for ASTC image\n"); - if (transfer == KHR_DF_TRANSFER_SRGB) { - assert(primaries == KHR_DF_PRIMARIES_SRGB && "Not a valid sRGB image\n"); - } + assert(KHR_DFDVAL(prototype->pDfd+1, MODEL) == KHR_DF_MODEL_ASTC + && "Invalid dfd generated for ASTC image\n"); + assert((transfer == KHR_DF_TRANSFER_SRGB + ? KHR_DFDVAL(prototype->pDfd+1, PRIMARIES) == KHR_DF_PRIMARIES_SRGB + : true) && "Not a valid sRGB image\n"); // Fix up the current (This) texture #undef DECLARE_PRIVATE @@ -850,10 +844,10 @@ ktxTexture_CompressAstcEx(ktxTexture* _This, ktxAstcParams* params) { memcpy(This->_private->_levelIndex, protoPriv._levelIndex, This->numLevels * sizeof(ktxLevelIndexEntry)); // Move the DFD and data from the prototype to This. - delete This->pDfd; + free(This->pDfd); This->pDfd = prototype->pDfd; prototype->pDfd = 0; - delete This->pData; + free(This->pData); This->pData = prototype->pData; This->dataSize = prototype->dataSize; prototype->pData = 0; diff --git a/lib/basis_encode.cpp b/lib/basis_encode.cpp index 038a391775..24b1d62399 100644 --- a/lib/basis_encode.cpp +++ b/lib/basis_encode.cpp @@ -381,7 +381,7 @@ static bool basisuEncoderInitialized = false; * @exception KTX_INVALID_OPERATION * The texture's images are supercompressed. * @exception KTX_INVALID_OPERATION - * The texture's image are in a block compressed + * The texture's images are in a block compressed * format. * @exception KTX_INVALID_OPERATION * The texture image's format is a packed format diff --git a/lib/basis_transcode.cpp b/lib/basis_transcode.cpp index a095ee723d..a11747169e 100644 --- a/lib/basis_transcode.cpp +++ b/lib/basis_transcode.cpp @@ -358,10 +358,10 @@ ktxTexture2_transcodeUastc(ktxTexture2* This, memcpy(priv._levelIndex, protoPriv._levelIndex, This->numLevels * sizeof(ktxLevelIndexEntry)); // Move the DFD and data from the prototype to This. - delete This->pDfd; + free(This->pDfd); This->pDfd = prototype->pDfd; prototype->pDfd = 0; - delete This->pData; + free(This->pData); This->pData = prototype->pData; This->dataSize = prototype->dataSize; prototype->pData = 0; diff --git a/tools/toktx/toktx.cc b/tools/toktx/toktx.cc index 7e5efa0b95..4a415b71c0 100644 --- a/tools/toktx/toktx.cc +++ b/tools/toktx/toktx.cc @@ -728,10 +728,12 @@ toktxApp::main(int argc, _TCHAR *argv[]) (options.astc && options.astcopts.mode != KTX_PACK_ASTC_ENCODER_MODE_HDR)); - if (options.astc && - options.astcopts.mode != KTX_PACK_ASTC_ENCODER_MODE_HDR && - image->getComponentSize() == 2) { - cerr << name << ": Warning! input file is 16bit but no HDR option provided." + if (options.astc && image->getComponentSize() == 2) + options.astcopts.mode = KTX_PACK_ASTC_ENCODER_MODE_HDR; + + if (options.astc && image->getComponentSize() != 2 && + options.astcopts.mode == KTX_PACK_ASTC_ENCODER_MODE_HDR) { + cerr << name << ": Warning! input file is not 16bit but HDR option is provided." << endl; } @@ -1255,6 +1257,9 @@ toktxApp::main(int argc, _TCHAR *argv[]) options.bopts.inputSwizzle[i] = defaultSwizzle[i]; } } + + bopts.threadCount = options.threadCount; + #if TRAVIS_DEBUG bopts.print(); #endif @@ -1281,13 +1286,15 @@ toktxApp::main(int argc, _TCHAR *argv[]) } } - if (firstImageOETF == KHR_DF_TRANSFER_SRGB) { + if (chosenOETF == KHR_DF_TRANSFER_SRGB) { astcopts.function = KTX_PACK_ASTC_ENCODER_FUNCTION_SRGB; } else { astcopts.function = KTX_PACK_ASTC_ENCODER_FUNCTION_LINEAR; } + astcopts.threadCount = options.threadCount; + ret = ktxTexture_CompressAstcEx(texture, &astcopts); if (KTX_SUCCESS != ret) { fprintf(stderr, "%s failed to compress KTX file \"%s\" to astc; KTX error: %s\n", diff --git a/utils/scapp.h b/utils/scapp.h index e607d844ec..57077e2b13 100644 --- a/utils/scapp.h +++ b/utils/scapp.h @@ -290,10 +290,14 @@ class scApp : public ktxApp { int zcmp; int astc; clamped zcmpLevel; + clamped threadCount; struct basisOptions bopts; struct astcOptions astcopts; - commandOptions() : zcmpLevel(ZSTD_CLEVEL_DEFAULT, 1U, 22U) { + commandOptions() : + zcmpLevel(ZSTD_CLEVEL_DEFAULT, 1U, 22U), + threadCount(max(1U, thread::hardware_concurrency()) , 1U, 10000U) + { ktx2 = false; etc1s = false; zcmp = false; @@ -331,8 +335,8 @@ class scApp : public ktxApp { { cerr << " --encode \n" - " Compress the image data with ASTC format or high-quality transcodable\n" - " ETC1S / BasisLZ or UASTC format. Implies --t2 for all encoding options.\n" + " Compress the image data to ASTC, transcodable ETC1S / BasisLZ or\n" + " high-quality transcodable UASTC format. Implies --t2.\n" " With each encoding option the following encoder specific options\n" " become valid, otherwise they are ignored.\n\n" " astc:\n" @@ -356,7 +360,8 @@ class scApp : public ktxApp { " 4x4x4: 2.00 bpp 6x6x5: 0.71 bpp\n" " 5x4x4: 1.60 bpp 6x6x6: 0.59 bpp\n" " --astc_mode \n" - " Specify which encoding mode to use. LDR is the default.\n" + " Specify which encoding mode to use. LDR is the default unless the input.\n" + " image is 16-bit in which case the default is HDR.\n" " --astc_quality \n" " The quality level configures the quality-performance tradeoff for\n" " the compressor; more complete searches of the search space improve\n" @@ -370,16 +375,6 @@ class scApp : public ktxApp { " medium | (equivalent to quality = 60)\n" " thorough | (equivalent to quality = 98)\n" " exhaustive | (equivalent to quality = 100)\n" - " --astc_normal\n" - " The input texture is a three component linear LDR normal map\n" - " storing unit length normals as (R=X, G=Y, B=Z). The output will\n" - " be a two component X+Y normal map stored as (RGB=X, A=Y),\n" - " optimized for angular error instead of simple PSNR. The Z\n" - " component can be recovered programmatically in shader code by\n" - " using the equation:\n\n" - " nml.xy = texture(...).ga; // Load in [0,1]\n" - " nml.xy = nml.xy * 2.0 - 1.0; // Unpack to [-1,1]\n" - " nml.z = sqrt(1 - dot(nml.xy, nml.xy)); // Compute Z\n" " etc1s:\n" " Supercompress the image data with ETC1S / BasisLZ.\n" " RED images will become RGB with RED in each component. RG images\n" @@ -424,9 +419,6 @@ class scApp : public ktxApp { " Set selector RDO quality threshold. The default is 1.25. Lower\n" " is higher quality but less quality per output bit (try\n" " [1.0,3.0]). This will override the value chosen by --qlevel.\n" - " --normal_map\n" - " Tunes codec parameters for better quality on normal maps (no\n" - " selector RDO, no endpoint RDO). Only valid for linear textures.\n" " --separate_rg_to_color_alpha\n" " Separates the input R and G channels to RGB and A (for tangent\n" " space XY normal maps). Only needed with 3 or 4 component input\n" @@ -485,6 +477,19 @@ class scApp : public ktxApp { " --uastc_rdo_m\n" " Disable RDO multithreading (slightly higher compression,\n" " deterministic).\n\n" + " --normal_mode\n" + " For ASTC encoder '--encode astc' assumes the input texture is\n" + " a three component linear LDR normal map storing unit length\n" + " normals as (R=X, G=Y, B=Z). The output will be a two component\n" + " X+Y normal map stored as (RGB=X, A=Y), optimized for angular\n" + " error instead of simple PSNR. The Z component can be recovered\n" + " programmatically in shader code by using the equation:\n\n" + " nml.xy = texture(...).ga; // Load in [0,1]\n" + " nml.xy = nml.xy * 2.0 - 1.0; // Unpack to [-1,1]\n" + " nml.z = sqrt(1 - dot(nml.xy, nml.xy)); // Compute Z\n\n" + " For ETC1S encoder '--encode etc1s' tunes codec parameters for \n" + " better quality on normal maps (no selector RDO, no endpoint RDO).\n" + " Only valid for linear textures.\n" " --no_sse Forbid use of the SSE instruction set. Ignored if CPU does not\n" " support SSE. Only the Basis Universal compressor uses SSE.\n" " --bcmp\n" @@ -522,7 +527,6 @@ scApp::scApp(string& version, string& defaultVersion, : ktxApp(version, defaultVersion, options), options(options) { argparser::option my_option_list[] = { - { "encode", argparser::option::required_argument, NULL, 'o' }, { "zcmp", argparser::option::optional_argument, NULL, 'z' }, { "no_multithreading", argparser::option::no_argument, NULL, 'N' }, { "threads", argparser::option::required_argument, NULL, 't' }, @@ -532,7 +536,7 @@ scApp::scApp(string& version, string& defaultVersion, { "endpoint_rdo_threshold", argparser::option::required_argument, NULL, 'E' }, { "max_selectors", argparser::option::required_argument, NULL, 'u' }, { "selector_rdo_threshold", argparser::option::required_argument, NULL, 'S' }, - { "normal_map", argparser::option::no_argument, NULL, 'n' }, + { "normal_mode", argparser::option::no_argument, NULL, 'n' }, { "separate_rg_to_color_alpha", argparser::option::no_argument, NULL, 1000 }, { "no_endpoint_rdo", argparser::option::no_argument, NULL, 1001 }, { "no_selector_rdo", argparser::option::no_argument, NULL, 1002 }, @@ -548,7 +552,7 @@ scApp::scApp(string& version, string& defaultVersion, { "astc_blk_d", argparser::option::required_argument, NULL, 1012 }, { "astc_mode", argparser::option::required_argument, NULL, 1013 }, { "astc_quality", argparser::option::required_argument, NULL, 1014 }, - { "astc_normal", argparser::option::no_argument, NULL, 1015 }, + { "encode", argparser::option::required_argument, NULL, 1015 }, // Deprecated options { "bcmp", argparser::option::no_argument, NULL, 'b' }, { "uastc", argparser::option::optional_argument, NULL, 1016 } @@ -557,7 +561,7 @@ scApp::scApp(string& version, string& defaultVersion, / sizeof(argparser::option); option_list.insert(option_list.begin(), my_option_list, my_option_list + lastOptionIndex); - short_opts += "oz;Nt:c:q:e:E:u:S:nb"; + short_opts += "z;Nt:c:q:e:E:u:S:nb"; } void @@ -597,10 +601,6 @@ scApp::processOption(argparser& parser, int opt) bool capture = true; switch (opt) { - case 'o': - setEncoder(parser.optarg); - options.ktx2 = 1; - break; case 'z': if (options.etc1s) { cerr << "Only one of '--encode etc1s|--bcmp' and --zcmp can be specified." @@ -628,11 +628,12 @@ scApp::processOption(argparser& parser, int opt) hasArg = true; break; case 'N': - options.bopts.threadCount = 1; + options.threadCount = 1; capture = false; break; case 'n': options.bopts.normalMap = 1; + options.astcopts.normalMap = true; break; case 1001: options.bopts.noEndpointRDO = 1; @@ -656,8 +657,7 @@ scApp::processOption(argparser& parser, int opt) hasArg = true; break; case 't': - options.bopts.threadCount = strtoi(parser.optarg.c_str()); - options.astcopts.threadCount = strtoi(parser.optarg.c_str()); + options.threadCount = strtoi(parser.optarg.c_str()); capture = false; break; case 1003: @@ -719,9 +719,6 @@ scApp::processOption(argparser& parser, int opt) options.astcopts.qualityLevel = astcQualityLevel(parser.optarg.c_str()); hasArg = true; break; - case 1015: // astc_normal - options.astcopts.normalMap = true; - break; case 'b': if (options.zcmp) { cerr << "Only one of --bcmp and --zcmp can be specified.\n" @@ -740,6 +737,10 @@ scApp::processOption(argparser& parser, int opt) options.etc1s = 1; options.ktx2 = 1; break; + case 1015: + setEncoder(parser.optarg); + options.ktx2 = 1; + break; case 1016: if (options.etc1s) { cerr << "Only one of `--encode etc1s|--bcmp` and `--uastc []` can be specified." From d440dd12198d0a13ea0f1e83e0afdd6d7b4e9e34 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Tue, 25 May 2021 19:23:25 +0100 Subject: [PATCH 14/65] Add combined normal mode --- tools/toktx/toktx.cc | 2 ++ utils/scapp.h | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/toktx/toktx.cc b/tools/toktx/toktx.cc index 4a415b71c0..035376acdb 100644 --- a/tools/toktx/toktx.cc +++ b/tools/toktx/toktx.cc @@ -1259,6 +1259,7 @@ toktxApp::main(int argc, _TCHAR *argv[]) } bopts.threadCount = options.threadCount; + bopts.normalMap = options.normalMode; #if TRAVIS_DEBUG bopts.print(); @@ -1294,6 +1295,7 @@ toktxApp::main(int argc, _TCHAR *argv[]) } astcopts.threadCount = options.threadCount; + astcopts.normalMap = options.normalMode; ret = ktxTexture_CompressAstcEx(texture, &astcopts); if (KTX_SUCCESS != ret) { diff --git a/utils/scapp.h b/utils/scapp.h index 57077e2b13..5c8e31057e 100644 --- a/utils/scapp.h +++ b/utils/scapp.h @@ -289,6 +289,7 @@ class scApp : public ktxApp { int etc1s; int zcmp; int astc; + ktx_bool_t normalMode; clamped zcmpLevel; clamped threadCount; struct basisOptions bopts; @@ -302,6 +303,7 @@ class scApp : public ktxApp { etc1s = false; zcmp = false; astc = false; + normalMode = false; } }; @@ -632,8 +634,7 @@ scApp::processOption(argparser& parser, int opt) capture = false; break; case 'n': - options.bopts.normalMap = 1; - options.astcopts.normalMap = true; + options.normalMode = true; break; case 1001: options.bopts.noEndpointRDO = 1; From 8374a6988565a9c4ba280d667de5c305d3c67cb1 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Wed, 26 May 2021 15:31:27 +0100 Subject: [PATCH 15/65] Minor typo fixes and ensures numLevels never less than 1 --- include/ktx.h | 2 +- lib/astc_encode.cpp | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/ktx.h b/include/ktx.h index 7432c549b8..647926bf14 100644 --- a/include/ktx.h +++ b/include/ktx.h @@ -1016,7 +1016,7 @@ typedef enum ktx_pack_astc_encoder_function_e { /** * @~English - * @brief Options specifiying ASTC encoder profile mode + * @brief Options specifying ASTC encoder profile mode * This and function is used later to derive the profile. */ typedef enum ktx_pack_astc_encoder_mode_e { diff --git a/lib/astc_encode.cpp b/lib/astc_encode.cpp index 09eaf8e7d0..2191197614 100644 --- a/lib/astc_encode.cpp +++ b/lib/astc_encode.cpp @@ -675,6 +675,10 @@ ktxTexture_CompressAstcEx(ktxTexture* _This, ktxAstcParams* params) { VkFormat vkFormat = astcVkFormat(params->blockDimension, sRGB); + // This->numLevels = 0 not allowed for block compressed formats + // But just in case make sure its not zero + This->numLevels = MAX(1, This->numLevels); + // Create a prototype texture to use for calculating sizes in the target // format and, as useful side effects, provide us with a properly sized // data allocation and the DFD for the target format. @@ -738,10 +742,6 @@ ktxTexture_CompressAstcEx(ktxTexture* _This, ktxAstcParams* params) { } // Walk in reverse on levels so we don't have to do this later - // This->numLevels = 0 not allowed for block compressed formats - // But just in case make sure its not zero - This->numLevels = MAX(1, This->numLevels); - assert(prototype->dataSize && "Prototype texture size not initialized.\n"); if (!prototype->pData) { From d535c09a8a5a32470af8f2910ca5e0b3589cb6fb Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Wed, 26 May 2021 15:37:48 +0100 Subject: [PATCH 16/65] Minor typo fix --- lib/astc_encode.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/astc_encode.cpp b/lib/astc_encode.cpp index 2191197614..54c4c4cc98 100644 --- a/lib/astc_encode.cpp +++ b/lib/astc_encode.cpp @@ -610,7 +610,7 @@ launchThreads(int threadCount, void (*func)(int, int, void*), void *payload) { * @exception KTX_INVALID_OPERATION * The texture's images are supercompressed. * @exception KTX_INVALID_OPERATION - * The texture's image are in a block compressed + * The texture's images are in a block compressed * format. * @exception KTX_INVALID_OPERATION * The texture image's format is a packed format From eb3bcfa971843b1531187c7992fa5c5828c89777 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Wed, 26 May 2021 15:53:37 +0100 Subject: [PATCH 17/65] Change build system to prefer ISA_NONE if defined --- CMakeLists.txt | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a9637868ec..7f536510e6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -634,23 +634,25 @@ add_subdirectory(interface/basisu_c_binding) # If ISA_AVX2 fails to compile user must chose other x86 options # On arm based systems ISA_NEON is default -if(CPU_ARCHITECTURE STREQUAL x86_64 OR CPU_ARCHITECTURE STREQUAL x86) - if (${ISA_SSE41}) - set(ASTC_LIB_NAME astcenc-sse4.1-static) - elseif (${ISA_SSE2}) - set(ASTC_LIB_NAME astcenc-sse2-static) - else() - set(ISA_AVX2 ON) # AVX2 is the default - set(ASTC_LIB_NAME astcenc-avx2-static) - message("ASTCENC: Building for AVX2. If this fails to compile, define either ISA_SSE41 or ISA_SSE2") - endif() -elseif(CPU_ARCHITECTURE STREQUAL arm OR CPU_ARCHITECTURE STREQUAL armv7 - OR CPU_ARCHITECTURE STREQUAL armv8 OR CPU_ARCHITECTURE STREQUAL arm64) - set(ISA_NEON ON) - set(ASTC_LIB_NAME astcenc-neon-static) -else() +if (${ISA_NONE}) set(ISA_NONE ON) set(ASTC_LIB_NAME astcenc-none-static) +else() + if(CPU_ARCHITECTURE STREQUAL x86_64 OR CPU_ARCHITECTURE STREQUAL x86) + if (${ISA_SSE41}) + set(ASTC_LIB_NAME astcenc-sse4.1-static) + elseif (${ISA_SSE2}) + set(ASTC_LIB_NAME astcenc-sse2-static) + else() + set(ISA_AVX2 ON) # AVX2 is the default + set(ASTC_LIB_NAME astcenc-avx2-static) + message("ASTCENC: Building for AVX2. If this fails to compile, define either ISA_SSE41 or ISA_SSE2") + endif() + elseif(CPU_ARCHITECTURE STREQUAL arm OR CPU_ARCHITECTURE STREQUAL armv7 + OR CPU_ARCHITECTURE STREQUAL armv8 OR CPU_ARCHITECTURE STREQUAL arm64) + set(ISA_NEON ON) + set(ASTC_LIB_NAME astcenc-neon-static) + endif() endif() # astcenc From 155f3d368aeb398eea4402fec0a28e36ac64ddf6 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Fri, 28 May 2021 22:25:54 +0100 Subject: [PATCH 18/65] Fix CI build issues for android and iOS, one has to specify -DARCH=aarch64 --- CMakeLists.txt | 3 +++ ci_scripts/build_android.sh | 3 ++- ci_scripts/build_android_debug.sh | 3 ++- ci_scripts/build_macos.sh | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7f536510e6..9d3f1daeed 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -659,6 +659,9 @@ endif() add_subdirectory(lib/astc-encoder) set_property(TARGET ${ASTC_LIB_NAME} PROPERTY POSITION_INDEPENDENT_CODE ON) target_link_libraries(ktx PRIVATE ${ASTC_LIB_NAME}) +# Since astcenc uses LTO in release build by default, turn it on for ktx too +# Its a good optimisation anyways and most compilers now support it +set_property(TARGET ktx PROPERTY INTERPROCEDURAL_OPTIMIZATION_RELEASE True) # Tools if(KTX_FEATURE_TOOLS) diff --git a/ci_scripts/build_android.sh b/ci_scripts/build_android.sh index 0ae79282f1..633c27692c 100755 --- a/ci_scripts/build_android.sh +++ b/ci_scripts/build_android.sh @@ -18,7 +18,8 @@ cmake . -G Ninja -B "build-android-$ANDROID_ABI" \ -DCMAKE_TOOLCHAIN_FILE="$ANDROID_NDK/build/cmake/android.toolchain.cmake" \ -DCMAKE_BUILD_TYPE=Release \ -DBASISU_SUPPORT_SSE=OFF \ --DISA_NONE=ON +-DISA_NEON=ON \ +-DARCH=aarch64 pushd "build-android-$ANDROID_ABI" diff --git a/ci_scripts/build_android_debug.sh b/ci_scripts/build_android_debug.sh index 66698c1fc7..28c3fd7f75 100755 --- a/ci_scripts/build_android_debug.sh +++ b/ci_scripts/build_android_debug.sh @@ -18,7 +18,8 @@ cmake . -G Ninja -B "build-android-$ANDROID_ABI-debug" \ -DCMAKE_TOOLCHAIN_FILE="$ANDROID_NDK/build/cmake/android.toolchain.cmake" \ -DCMAKE_BUILD_TYPE=Debug \ -DBASISU_SUPPORT_SSE=OFF \ --DISA_NONE=ON +-DISA_NEON=ON \ +-DARCH=aarch64 pushd "build-android-$ANDROID_ABI-debug" diff --git a/ci_scripts/build_macos.sh b/ci_scripts/build_macos.sh index b1f2dfa779..25195944c7 100755 --- a/ci_scripts/build_macos.sh +++ b/ci_scripts/build_macos.sh @@ -86,7 +86,7 @@ popd # echo "Configure KTX-Software (iOS)" -cmake -GXcode -Bbuild-ios -DCMAKE_SYSTEM_NAME=iOS -DKTX_FEATURE_LOADTEST_APPS=ON -DKTX_FEATURE_DOC=ON -DVULKAN_INSTALL_DIR="${VULKAN_INSTALL_DIR}" +cmake -GXcode -Bbuild-ios -DARCH=aarch64 -DCMAKE_SYSTEM_NAME=iOS -DKTX_FEATURE_LOADTEST_APPS=ON -DKTX_FEATURE_DOC=ON -DVULKAN_INSTALL_DIR="${VULKAN_INSTALL_DIR}" pushd build-ios echo "Build KTX-Software (iOS Debug)" cmake --build . --config Debug -- -sdk iphoneos CODE_SIGN_IDENTITY="" CODE_SIGNING_ALLOWED=NO CODE_SIGNING_REQUIRED=NO | tee -a fullbuild.log | xcpretty From e1a52709caefda09e496591bf1cfaf16c670b4dd Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Mon, 31 May 2021 14:25:34 +0100 Subject: [PATCH 19/65] Add ci fix for android aarch32 --- CMakeLists.txt | 4 ++++ ci_scripts/build_android.sh | 4 +--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9d3f1daeed..b2b84aa4bf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -652,6 +652,10 @@ else() OR CPU_ARCHITECTURE STREQUAL armv8 OR CPU_ARCHITECTURE STREQUAL arm64) set(ISA_NEON ON) set(ASTC_LIB_NAME astcenc-neon-static) + set(ARCH aarch64) + if(CPU_ARCHITECTURE STREQUAL arm OR CPU_ARCHITECTURE STREQUAL armv7) + set(ARCH aarch32) + endif() endif() endif() diff --git a/ci_scripts/build_android.sh b/ci_scripts/build_android.sh index 633c27692c..42408cb544 100755 --- a/ci_scripts/build_android.sh +++ b/ci_scripts/build_android.sh @@ -17,9 +17,7 @@ cmake . -G Ninja -B "build-android-$ANDROID_ABI" \ -DANDROID_NDK="$ANDROID_NDK" \ -DCMAKE_TOOLCHAIN_FILE="$ANDROID_NDK/build/cmake/android.toolchain.cmake" \ -DCMAKE_BUILD_TYPE=Release \ --DBASISU_SUPPORT_SSE=OFF \ --DISA_NEON=ON \ --DARCH=aarch64 +-DBASISU_SUPPORT_SSE=OFF pushd "build-android-$ANDROID_ABI" From 59329b4fb05965ae8a06761e775bd928d2ec0a49 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Mon, 31 May 2021 15:22:07 +0100 Subject: [PATCH 20/65] Add ci fix for android aarch32 --- CMakeLists.txt | 9 ++++----- ci_scripts/build_android_debug.sh | 4 +--- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b2b84aa4bf..903a5f49f5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -644,17 +644,16 @@ else() elseif (${ISA_SSE2}) set(ASTC_LIB_NAME astcenc-sse2-static) else() - set(ISA_AVX2 ON) # AVX2 is the default + set(ISA_AVX2 ON) set(ASTC_LIB_NAME astcenc-avx2-static) - message("ASTCENC: Building for AVX2. If this fails to compile, define either ISA_SSE41 or ISA_SSE2") endif() elseif(CPU_ARCHITECTURE STREQUAL arm OR CPU_ARCHITECTURE STREQUAL armv7 OR CPU_ARCHITECTURE STREQUAL armv8 OR CPU_ARCHITECTURE STREQUAL arm64) set(ISA_NEON ON) set(ASTC_LIB_NAME astcenc-neon-static) - set(ARCH aarch64) - if(CPU_ARCHITECTURE STREQUAL arm OR CPU_ARCHITECTURE STREQUAL armv7) - set(ARCH aarch32) + set(ARCH aarch64 CACHE STRING "Target architecture") + if(CPU_ARCHITECTURE STREQUAL armv7) + set(ARCH aarch32 CACHE STRING "Target architecture" FORCE) endif() endif() endif() diff --git a/ci_scripts/build_android_debug.sh b/ci_scripts/build_android_debug.sh index 28c3fd7f75..dfa79e8fc6 100755 --- a/ci_scripts/build_android_debug.sh +++ b/ci_scripts/build_android_debug.sh @@ -17,9 +17,7 @@ cmake . -G Ninja -B "build-android-$ANDROID_ABI-debug" \ -DANDROID_NDK="$ANDROID_NDK" \ -DCMAKE_TOOLCHAIN_FILE="$ANDROID_NDK/build/cmake/android.toolchain.cmake" \ -DCMAKE_BUILD_TYPE=Debug \ --DBASISU_SUPPORT_SSE=OFF \ --DISA_NEON=ON \ --DARCH=aarch64 +-DBASISU_SUPPORT_SSE=OFF pushd "build-android-$ANDROID_ABI-debug" From 98ff1c6d396a9f2cb7dc6e559d2d46ac619ae739 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Tue, 1 Jun 2021 15:21:58 +0100 Subject: [PATCH 21/65] Add ci fix for android x86/x86_64 --- CMakeLists.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6ef9a1c196..720ad56064 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -658,6 +658,12 @@ else() set(ISA_AVX2 ON) set(ASTC_LIB_NAME astcenc-avx2-static) endif() + if(CPU_ARCHITECTURE STREQUAL x86) + set(ISA_NONE ON) + set(ISA_AVX2 OFF) + set(ASTCENC_POPCNT 0) + set(ASTC_LIB_NAME astcenc-none-static) + endif() elseif(CPU_ARCHITECTURE STREQUAL arm OR CPU_ARCHITECTURE STREQUAL armv7 OR CPU_ARCHITECTURE STREQUAL armv8 OR CPU_ARCHITECTURE STREQUAL arm64) set(ISA_NEON ON) From 0d4e47362765d597a6abad0bc9e8b117d74ecd13 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Wed, 2 Jun 2021 11:22:24 +0100 Subject: [PATCH 22/65] Fix rescale argument, add a 3 valued enum rescale_e(eNoRescale, eAlwaysRescaleTo8Bits, eRescaleTo8BitsIfLess) also adds the appropriate changes into image loaders and toktx.cc --- lib/astc_encode.cpp | 6 ------ tools/toktx/image.cc | 4 ++-- tools/toktx/image.hpp | 13 ++++++++----- tools/toktx/jpgimage.cc | 2 +- tools/toktx/npbmimage.cc | 26 +++++++++++++------------- tools/toktx/pngimage.cc | 8 +++++--- tools/toktx/toktx.cc | 10 +++++++--- 7 files changed, 36 insertions(+), 33 deletions(-) diff --git a/lib/astc_encode.cpp b/lib/astc_encode.cpp index 54c4c4cc98..444d5ceadb 100644 --- a/lib/astc_encode.cpp +++ b/lib/astc_encode.cpp @@ -616,9 +616,6 @@ launchThreads(int threadCount, void (*func)(int, int, void*), void *payload) { * The texture image's format is a packed format * (e.g. RGB565). * @exception KTX_INVALID_OPERATION - * The texture image format's component size is not - 8-bits. - * @exception KTX_INVALID_OPERATION * The texture's images are 1D. Only 2D images can * be supercompressed. * @exception KTX_INVALID_OPERATION @@ -656,9 +653,6 @@ ktxTexture_CompressAstcEx(ktxTexture* _This, ktxAstcParams* params) { uint32_t num_components, component_size; getDFDComponentInfoUnpacked(This->pDfd, &num_components, &component_size); - if (component_size != 1) - return KTX_INVALID_OPERATION; // Can only deal with 8-bit components - if (This->pData == NULL) { result = ktxTexture2_LoadImageData((ktxTexture2*)This, nullptr, 0); diff --git a/tools/toktx/image.cc b/tools/toktx/image.cc index c6314a29c6..2f5706821f 100644 --- a/tools/toktx/image.cc +++ b/tools/toktx/image.cc @@ -27,7 +27,7 @@ const std::vector Image::CreateFunctions = { }; Image* Image::CreateFromFile(const _tstring& name, - bool transformOETF, bool rescaleTo8Bit) { + bool transformOETF, rescale_e rescale) { FILE* f; Image* image; @@ -44,7 +44,7 @@ Image* Image::CreateFromFile(const _tstring& name, func = CreateFunctions.begin(); for (; func < CreateFunctions.end(); func++ ) { try { - image = (*func)(f, transformOETF, rescaleTo8Bit); + image = (*func)(f, transformOETF, rescale); return image; } catch (different_format&) { rewind(f); diff --git a/tools/toktx/image.hpp b/tools/toktx/image.hpp index 0a234bf831..8f47fc48c0 100644 --- a/tools/toktx/image.hpp +++ b/tools/toktx/image.hpp @@ -267,6 +267,9 @@ class Image { enum colortype_e { eLuminance=0, eLuminanceAlpha=1, eR=2, eRG, eRGB, eRGBA }; + enum rescale_e { + eNoRescale, eAlwaysRescaleTo8Bits, eRescaleTo8BitsIfLess + }; virtual ~Image() { }; @@ -283,18 +286,18 @@ class Image { } typedef Image* (*CreateFunction)(FILE* f, bool transformOETF, - bool rescaleTo8Bitbool); + rescale_e rescale); static const std::vector CreateFunctions; static Image* CreateFromNPBM(FILE*, bool transformOETF = true, - bool rescaleTo8Bitbool = false); + rescale_e rescale = eNoRescale); static Image* CreateFromJPG(FILE* f, bool transformOETF = true, - bool rescaleTo8Bitbool = false); + rescale_e rescale = eNoRescale); static Image* CreateFromPNG(FILE* f, bool transformOETF = true, - bool rescaleTo8Bitbool = false); + rescale_e rescale = eNoRescale); static Image* CreateFromFile(const _tstring& name, bool transformOETF = true, - bool rescaleTo8Bitbool = false); + rescale_e rescale = eNoRescale); virtual operator uint8_t*() = 0; diff --git a/tools/toktx/jpgimage.cc b/tools/toktx/jpgimage.cc index b0b660023a..2a6f5f9d36 100644 --- a/tools/toktx/jpgimage.cc +++ b/tools/toktx/jpgimage.cc @@ -74,7 +74,7 @@ class myjpgdstream : public jpeg_decoder_file_stream { // All JPEG files are sRGB. Image* -Image::CreateFromJPG(FILE* src, bool, bool) +Image::CreateFromJPG(FILE* src, bool, rescale_e) { myjpgdstream stream(src); uint32_t componentCount; diff --git a/tools/toktx/npbmimage.cc b/tools/toktx/npbmimage.cc index f7d700c1cb..f18271a766 100644 --- a/tools/toktx/npbmimage.cc +++ b/tools/toktx/npbmimage.cc @@ -93,9 +93,9 @@ void skipNonData(FILE *src) skipSpaces(src); } -static Image* createFromPPM(FILE*, bool transformOETF, bool rescaleTo8Bits); -static Image* createFromPGM(FILE*, bool transformOETF, bool rescaleTo8Bits); -static Image* createFromPAM(FILE*, bool transformOETF, bool rescaleTo8Bits); +static Image* createFromPPM(FILE*, bool transformOETF, Image::rescale_e rescale); +static Image* createFromPGM(FILE*, bool transformOETF, Image::rescale_e rescale); +static Image* createFromPAM(FILE*, bool transformOETF, Image::rescale_e rescale); static void parseHeader(FILE* src, uint32_t& width, uint32_t& height, int32_t& maxval); static void readImage(FILE* src, Image& image, int32_t maxval); @@ -120,7 +120,7 @@ static void readImage(FILE* src, Image& image, int32_t maxval); //! @author Mark Callow //! Image* -Image::CreateFromNPBM(FILE* src, bool transformOETF, bool rescaleTo8Bits) +Image::CreateFromNPBM(FILE* src, bool transformOETF, Image::rescale_e rescale) { char line[255]; int numvals; @@ -130,11 +130,11 @@ Image::CreateFromNPBM(FILE* src, bool transformOETF, bool rescaleTo8Bits) numvals = fscanf(src, "%3s", line); if (numvals != 0) { if (strcmp(line, "P6") == 0) { - return createFromPPM(src, transformOETF, rescaleTo8Bits); + return createFromPPM(src, transformOETF, rescale); } else if (strcmp(line, "P5") == 0) { - return createFromPGM(src, transformOETF, rescaleTo8Bits); + return createFromPGM(src, transformOETF, rescale); } else if (strcmp(line, "P7") == 0) { - return createFromPAM(src, transformOETF, rescaleTo8Bits); + return createFromPAM(src, transformOETF, rescale); } else if (strcmp(line, "P3") == 0) { throw std::runtime_error("Plain PPM format is not supported."); } @@ -170,7 +170,7 @@ Image::CreateFromNPBM(FILE* src, bool transformOETF, bool rescaleTo8Bits) //! @author Mark Callow //! Image* -createFromPPM(FILE* src, bool transformOETF, bool rescaleTo8Bits) +createFromPPM(FILE* src, bool transformOETF, Image::rescale_e rescale) { int32_t maxval; uint32_t width, height; @@ -182,7 +182,7 @@ createFromPPM(FILE* src, bool transformOETF, bool rescaleTo8Bits) //fprintf(stderr, "maxval is %d\n",maxval); // PPM is 3 components - if (maxval > 255 && !rescaleTo8Bits) + if (maxval > 255 && rescale != Image::rescale_e::eAlwaysRescaleTo8Bits) image = new rgb16image(width, height); else image = new rgb8image(width, height); @@ -232,7 +232,7 @@ createFromPPM(FILE* src, bool transformOETF, bool rescaleTo8Bits) //! @author Mark Callow //! Image* -createFromPGM(FILE* src, bool transformOETF, bool rescaleTo8Bits) +createFromPGM(FILE* src, bool transformOETF, Image::rescale_e rescale) { int maxval; uint32_t width, height; @@ -243,7 +243,7 @@ createFromPGM(FILE* src, bool transformOETF, bool rescaleTo8Bits) parseHeader(src, width, height, maxval); // PGM is 1 component. Treat as luminance for consistency with .png & .jpg. - if (maxval > 255 && !rescaleTo8Bits) + if (maxval > 255 && rescale != Image::rescale_e::eAlwaysRescaleTo8Bits) image = new r16image(width, height); else image = new r8image(width, height); @@ -297,7 +297,7 @@ createFromPGM(FILE* src, bool transformOETF, bool rescaleTo8Bits) //! @author Mark Callow //! Image* -createFromPAM(FILE* src, bool transformOETF, bool rescaleTo8Bits) +createFromPAM(FILE* src, bool transformOETF, Image::rescale_e rescale) { char line[255]; #define MAX_TUPLETYPE_SIZE 20 @@ -343,7 +343,7 @@ createFromPAM(FILE* src, bool transformOETF, bool rescaleTo8Bits) if (maxval <= 0 || maxval >= (1<<16)) { throw Image::invalid_file("Max color component value must be > 0 && < 65536."); } - if (maxval > 255 && !rescaleTo8Bits) { + if (maxval > 255 && rescale != Image::rescale_e::eAlwaysRescaleTo8Bits) { switch (depth) { case 1: image = new r16image(width, height); diff --git a/tools/toktx/pngimage.cc b/tools/toktx/pngimage.cc index 0144876d37..90d49dc1c2 100644 --- a/tools/toktx/pngimage.cc +++ b/tools/toktx/pngimage.cc @@ -28,7 +28,7 @@ void warning(const char *pFmt, ...); Image* -Image::CreateFromPNG(FILE* src, bool transformOETF, bool rescaleTo8Bits) +Image::CreateFromPNG(FILE* src, bool transformOETF, Image::rescale_e rescale) { // Unfortunately LoadPNG doesn't believe in stdio plus // the function we need only reads from memory. To avoid @@ -104,7 +104,7 @@ Image::CreateFromPNG(FILE* src, bool transformOETF, bool rescaleTo8Bits) case LCT_GREY: componentCount = 1; // TODO: Create 4-bit color type and rescale 1- & 2-bpp to that. - rescaleTo8Bits = true; + rescale = Image::eAlwaysRescaleTo8Bits; break; case LCT_RGB: if (pTrnsChunk != nullptr) { @@ -138,7 +138,9 @@ Image::CreateFromPNG(FILE* src, bool transformOETF, bool rescaleTo8Bits) // To avoid potentially uninitialized variable warning. componentCount = 0; } - if (rescaleTo8Bits) { + if (rescale == eAlwaysRescaleTo8Bits + || (rescale == eRescaleTo8BitsIfLess + && state.info_png.color.bitdepth < 8)) { state.info_raw.bitdepth = 8; if (state.info_png.color.bitdepth != 8) { warning("Rescaling %d-bit image to 8 bits.", diff --git a/tools/toktx/toktx.cc b/tools/toktx/toktx.cc index 035376acdb..ccea7c9655 100644 --- a/tools/toktx/toktx.cc +++ b/tools/toktx/toktx.cc @@ -721,12 +721,16 @@ toktxApp::main(int argc, _TCHAR *argv[]) Image* image; try { + Image::rescale_e rescale = Image::eNoRescale; + if (options.etc1s || options.bopts.uastc) + rescale = Image::rescale_e::eAlwaysRescaleTo8Bits; + else if (options.astc) + rescale = Image::rescale_e::eRescaleTo8BitsIfLess; + image = Image::CreateFromFile(infile, options.assign_oetf == KHR_DF_TRANSFER_UNSPECIFIED, - options.etc1s || options.bopts.uastc || - (options.astc && - options.astcopts.mode != KTX_PACK_ASTC_ENCODER_MODE_HDR)); + rescale); if (options.astc && image->getComponentSize() == 2) options.astcopts.mode = KTX_PACK_ASTC_ENCODER_MODE_HDR; From d65be83aa9932948d22c03de5c9a3f92358ae1fe Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Wed, 2 Jun 2021 11:33:52 +0100 Subject: [PATCH 23/65] Add warnings if input is > 8 bit while LDR option is chosen, either explicitly or the default --- tools/toktx/toktx.cc | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tools/toktx/toktx.cc b/tools/toktx/toktx.cc index ccea7c9655..40770325de 100644 --- a/tools/toktx/toktx.cc +++ b/tools/toktx/toktx.cc @@ -732,9 +732,15 @@ toktxApp::main(int argc, _TCHAR *argv[]) options.assign_oetf == KHR_DF_TRANSFER_UNSPECIFIED, rescale); - if (options.astc && image->getComponentSize() == 2) - options.astcopts.mode = KTX_PACK_ASTC_ENCODER_MODE_HDR; + // If input is > 8bit and user wants LDR issue quality loss warning + if (options.astc && image->getComponentSize() == 2 + && options.astcopts.mode == KTX_PACK_ASTC_ENCODER_MODE_LDR) { + cerr << name << ": Warning! input file is 16bit but LDR option is provided." + << " Expect quality loss in the output." + << endl; + } + // If input is 8bit and user wants HDR issue warning if (options.astc && image->getComponentSize() != 2 && options.astcopts.mode == KTX_PACK_ASTC_ENCODER_MODE_HDR) { cerr << name << ": Warning! input file is not 16bit but HDR option is provided." From f99d33c832d5dbbdf5c33d2f23c3b2135957e114 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Wed, 2 Jun 2021 12:35:52 +0100 Subject: [PATCH 24/65] Add updated html docs for scaap --- utils/scapp.h | 328 +++++++++++++++++++++++++++++++------------------- 1 file changed, 201 insertions(+), 127 deletions(-) diff --git a/utils/scapp.h b/utils/scapp.h index 5c8e31057e..0d1101d3e6 100644 --- a/utils/scapp.h +++ b/utils/scapp.h @@ -52,137 +52,209 @@ astcEncoderMode(const char* mode); // so the table below has to be laboriously done in html. //! [scApp options]
-
--bcmp
-
Supercompress the image data with ETC1S / BasisLZ. Implies @b --t2. - RED images will become RGB with RED in each component. RG images will - have R in the RGB part and G in the alpha part of the compressed - texture. When set, the following BasisLZ-related options become valid - otherwise they are ignored. -
+
--encode <astc|etc1s|uastc>
+
Compress the image data to ASTC, transcodable ETC1S / BasisLZ or + high-quality transcodable UASTC format. Implies @b --t2. + With each encoding option the following encoder specific options + become valid, otherwise they are ignored.
+
+
astc:
+
Create a texture in high-quality ASTC format.
+
--astc_blk_d <XxY|XxYxZ>
+
Specify which block dimension to use for compressing the textures. + e.g. @b --astc_blk_d 6x5 for 2D or @b --astc_blk_d 6x6x6 for 3D. + 6x6 is default for 2D. + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Supported 2D block dimensions are:
4x4 8.00 bpp
5x4 6.40 bpp
5x5 5.12 bpp
6x5 4.27 bpp
6x6 3.56 bpp
8x5 3.20 bpp
8x6 2.67 bpp
10x5 2.56 bpp
10x6 2.13 bpp
8x8 2.00 bpp
10x8 1.60 bpp
10x10 1.28 bpp
12x10 1.07 bpp
12x12 0.89 bpp
Supported 3D block dimensions are:
3x3x3 4.74 bpp
4x3x3 3.56 bpp
4x4x3 2.67 bpp
4x4x4 2.00 bpp
5x4x4 1.60 bpp
5x5x4 1.28 bpp
5x5x5 1.02 bpp
6x5x5 0.85 bpp
6x6x5 0.71 bpp
6x6x6 0.59 bpp
+
--astc_mode <ldr|hdr>
+
Specify which encoding mode to use. LDR is the default unless the input. + image is 16-bit in which case the default is HDR.
+
--astc_quality <level>
+
The quality level configures the quality-performance tradeoff for + the compressor; more complete searches of the search space improve + image quality at the expense of compression time. Default is 'medium' + The quality level can be set to fastest (0) and thorough (100) via the + following fixed quality presets: + + + + + + + +
Level Quality
fastest (equivalent to quality = 0)
fast (equivalent to quality = 10)
medium (equivalent to quality = 60)
thorough (equivalent to quality = 98)
exhaustive (equivalent to quality = 100)
+
+
+
+
etc1s:
+
Supercompress the image data with ETC1S / BasisLZ. + RED images will become RGB with RED in each component. RG images + will have R in the RGB part and G in the alpha part of the + compressed texture. When set, the following BasisLZ-related + options become valid, otherwise they are ignored.
--no_multithreading
-
Disable multithreading. Deprecated. For backward compatibility - only. Use @b --threads 1 instead.
-
--threads <count>
-
Explicitly set the number of threads to use during compression. - By default, ETC1S / BasisLZ compression will use the number of threads - reported by @c thread::hardware_concurrency or 1 if value - returned is 0.
-
--clevel <level>
-
ETC1S / BasisLZ compression level, an encoding speed vs. quality - tradeoff. Range is [0,5], default is 1. Higher values are slower, but - give higher quality.
-
--qlevel <level>
-
ETC1S / BasisLZ quality level. Range is [1,255]. Lower gives better - compression/lower quality/faster. Higher gives less compression - /higher quality/slower. @b --qlevel automatically determines values - for @b --max_endpoints, @b --max-selectors, - @b --endpoint_rdo_threshold and @b --selector_rdo_threshold for the - target quality level. Setting these options overrides the values - determined by @b --qlevel which defaults to 128 if neither it nor - both of @b --max_endpoints and @b --max_selectors have been set. - - @note Both of @b --max_endpoints and @b --max_selectors must be set - for them to have any effect. If all three options are set, a - warning will be issued that @b --qlevel will be ignored. - @note @b --qlevel will only determine values for - @b --endpoint_rdo_threshold and @b --selector_rdo_threshold - when its value exceeds 128, otherwise their defaults will be used. +
Disable multithreading. Deprecated. For backward compatibility. + Use @b --threads 1 instead.
+
--clevel
+
ETC1S / BasisLZ compression level, an encoding speed vs. quality + tradeoff. Range is [0,5], default is 1. Higher values are slower, + but give higher quality.
+
--qlevel
+
ETC1S / BasisLZ quality level. Range is [1,255]. Lower gives + better compression/lower quality/faster. Higher gives less + compression/higher quality/slower. @b --qlevel automatically + determines values for @b --max_endpoints, @b --max-selectors, + @b --endpoint_rdo_threshold and @b --selector_rdo_threshold for the + target quality level. Setting these options overrides the values + determined by -qlevel which defaults to 128 if neither it nor + both of @b --max_endpoints and @b --max_selectors have been set. + + Note that both of @b --max_endpoints and @b --max_selectors + must be set for them to have any effect. If all three options + are set, a warning will be issued that @b --qlevel will be ignored. + + Note also that @b --qlevel will only determine values for + @b --endpoint_rdo_threshold and @b --selector_rdo_threshold when + its value exceeds 128, otherwise their defaults will be used.
--max_endpoints <arg>
-
Manually set the maximum number of color endpoint clusters. - Range is [1,16128]. Default is 0, unset. If this is set, - @b --max_selectors must also be set, otherwise the value - will be ignored.
-
--endpoint_rdo_threshold <arg>
-
Set endpoint RDO quality threshold. The default is 1.25. Lower is - higher quality but less quality per output bit (try [1.0,3.0]). - This will override the value chosen by @b --qlevel.
+
Manually set the maximum number of color endpoint clusters. Range + is [1,16128]. Default is 0, unset.
+
--endpoint_rdo_threshold
+
Set endpoint RDO quality threshold. The default is 1.25. Lower + is higher quality but less quality per output bit (try + [1.0,3.0]). This will override the value chosen by @b --qlevel.
--max_selectors <arg>
-
Manually set the maximum number of color selector clusters. Range - is [1,16128]. Default is 0, unset. If this is set, - @b --max_selectors must also be set, otherwise the value - will be ignored.
-
--selector_rdo_threshold <arg>
-
Set selector RDO quality threshold. The default is 1.5. Lower is - higher quality but less quality per output bit (try [1.0,3.0]. - This will override the value chosen by @b --qlevel.
-
--normal_map
-
Tunes codec parameters for better quality on normal maps (no - selector RDO, no endpoint RDO). Only valid for linear textures.
+
Manually set the maximum number of color selector clusters from + [1,16128]. Default is 0, unset.
+
--selector_rdo_threshold
+
Set selector RDO quality threshold. The default is 1.25. Lower + is higher quality but less quality per output bit (try + [1.0,3.0]). This will override the value chosen by @b --qlevel.
--separate_rg_to_color_alpha
-
Separates the input R and G channels to RGB and A (for tangent - space XY normal maps). Only needed with 3 or 4 component input - images.
+
Separates the input R and G channels to RGB and A (for tangent + space XY normal maps). Only needed with 3 or 4 component input + images.
--no_endpoint_rdo
-
Disable endpoint rate distortion optimizations. Slightly faster, - less noisy output, but lower quality per output bit. Default is - to do endpoint RDO.
+
Disable endpoint rate distortion optimizations. Slightly faster, + less noisy output, but lower quality per output bit. Default is + to do endpoint RDO.
--no_selector_rdo
-
Disable selector rate distortion optimizations. Slightly faster, - less noisy output, but lower quality per output bit. Default is - to do selector RDO.
-
-
-
--uastc [<level>]
-
Create a texture in high-quality transcodable UASTC format. Implies - @b --t2. The optional parameter @e level selects a speed vs quality - tradeoff as shown in the following table: - - - - - - - - -
Level Speed Quality
0 Fastest 43.45dB
1 Faster 46.49dB
2 Default 47.47dB
3 Slower 48.01dB
4 Very slow 48.24dB
- - You are strongly encouraged to also specify @b --zcmp to losslessly - compress the UASTC data. This and any LZ-style compression can be made - more effective by conditioning the UASTC texture data using the - Rate Distortion Optimization (RDO) post-process stage. When @b --uastc - is set the following options become available for controlling RDO: -
-
--uastc_rdo_l [<lambda>]
-
Enable UASTC RDO post-processing and optionally set UASTC RDO - quality scalar (lambda) to @e lambda. Lower values yield higher - quality/larger LZ compressed files, higher values yield lower - quality/smaller LZ compressed files. A good range to try is - [.25,10]. For normal maps a good range is [.25-.75]. The full - range is [.001,10.0]. Default is 1.0. - - @note Previous versions used the @b --uastc_rdo_q option which was - removed because the RDO algorithm changed.
-
--uastc_rdo_d <dictsize>
-
Set UASTC RDO dictionary size in bytes. Default is 4096. Lower - values=faster, but give less compression. Range is [64,65536].
-
--uastc_rdo_b <scale>
-
Set UASTC RDO max smooth block error scale. Range is [1.0,300.0]. - Default is 10.0, 1.0 is disabled. Larger values suppress more - artifacts (and allocate more bits) on smooth blocks.
-
--uastc_rdo_s <deviation>
-
Set UASTC RDO max smooth block standard deviation. Range is - [.01,65536.0]. Default is 18.0. Larger values expand the range of - blocks considered smooth.
-
--uastc_rdo_f
-
Do not favor simpler UASTC modes in RDO mode.
-
--uastc_rdo_m
-
Disable RDO multithreading (slightly higher compression, - deterministic).
-
-
-
--no_sse
-
Forbid use of the SSE instruction set. Ignored if CPU does not - support SSE. Only the Basis Universal compressor uses SSE.
-
--verbose
-
Print encoder/compressor activity status to stdout. Currently only - the Basis Universal compressor emits status.
-
--zcmp [<compressionLevel>]
-
Supercompress the data with Zstandard. Implies @b --t2. Can be used - with data in any format except ETC1S / BasisLZ (@b --bcmp). Most - effective with RDO-conditioned UASTC or uncompressed formats. The - optional @e compressionLevel range is 1 - 22 and the default is 3. - Lower values=faster but give less compression. Values above 20 should - be used with caution as they require more memory.
+
Disable selector rate distortion optimizations. Slightly faster, + less noisy output, but lower quality per output bit. Default is + to do selector RDO.
+
+
+
uastc:
+
Create a texture in high-quality transcodable UASTC format.
+
--uastc_quality <level>
+
This optional parameter selects a speed vs quality + tradeoff as shown in the following table: + + + + + + + + +
Level Speed Quality
0 Fastest 43.45dB
1 Faster 46.49dB
2 Default 47.47dB
3 Slower 48.01dB
4 Very slow 48.24dB
+ + You are strongly encouraged to also specify @b --zcmp to losslessly + compress the UASTC data. This and any LZ-style compression can + be made more effective by conditioning the UASTC texture data + using the Rate Distortion Optimization (RDO) post-process stage. + When uastc encoding is set the following options become available + for controlling RDO:
+
--uastc_rdo_l [<lambda>]
+
Enable UASTC RDO post-processing and optionally set UASTC RDO + quality scalar (lambda) to @e lambda. Lower values yield higher + quality/larger LZ compressed files, higher values yield lower + quality/smaller LZ compressed files. A good range to try is + [.25,10]. For normal maps a good range is [.25,.75]. The full + range is [.001,10.0]. Default is 1.0. + + Note that previous versions used the @b --uastc_rdo_q option which + was removed because the RDO algorithm changed.
+
--uastc_rdo_d <dictsize>
+
Set UASTC RDO dictionary size in bytes. Default is 4096. Lower + values=faster, but give less compression. Range is [64,65536].
+
--uastc_rdo_b <scale>
+
Set UASTC RDO max smooth block error scale. Range is [1.0,300.0]. + Default is 10.0, 1.0 is disabled. Larger values suppress more + artifacts (and allocate more bits) on smooth blocks.
+
--uastc_rdo_s <deviation>
+
Set UASTC RDO max smooth block standard deviation. Range is + [.01,65536.0]. Default is 18.0. Larger values expand the range + of blocks considered smooth.
+
--uastc_rdo_f
+
Do not favor simpler UASTC modes in RDO mode.
+
--uastc_rdo_m
+
Disable RDO multithreading (slightly higher compression, + deterministic).
+
+
--normal_mode
+
For ASTC encoder '@b --encode astc' assumes the input texture is + a three component linear LDR normal map storing unit length + normals as (R=X, G=Y, B=Z). The output will be a two component + X+Y normal map stored as (RGB=X, A=Y), optimized for angular + error instead of simple PSNR. The Z component can be recovered + programmatically in shader code by using the equation: + 
+    nml.xy = texture(...).ga;              // Load in [0,1]
+    nml.xy = nml.xy * 2.0 - 1.0;           // Unpack to [-1,1]
+    nml.z = sqrt(1 - dot(nml.xy, nml.xy)); // Compute Z
+
+ For ETC1S encoder '@b --encode etc1s' tunes codec parameters for + better quality on normal maps (no selector RDO, no endpoint RDO). + Only valid for linear textures.
+
--no_sse
+
Forbid use of the SSE instruction set. Ignored if CPU does not + support SSE. Only the Basis Universal compressor uses SSE.
+
--bcmp
+
Deprecated. Use '@b --encode etc1s' instead.
+
--uastc [<level>]
+
Deprecated. Use '@b --encode uastc' instead.
+
--zcmp [<compressionLevel>]
+
Supercompress the data with Zstandard. Implies @b --t2. Can be used + with data in any format except ETC1S / BasisLZ. Most + effective with RDO-conditioned UASTC or uncompressed formats. The + optional compressionLevel range is 1 - 22 and the default is 3. + Lower values=faster but give less compression. Values above 20 + should be used with caution as they require more memory.
+
--threads <count>
+
Explicitly set the number of threads to use during compression. + By default, ETC1S / BasisLZ and ASTC compression will use the number of + threads reported by thread::hardware_concurrency or 1 if value + returned is 0.
+
--verbose
+
Print encoder/compressor activity status to stdout. Currently + only the astc, etc1s and uastc encoders emit status.
@snippet{doc} ktxapp.h ktxApp options @@ -492,7 +564,8 @@ class scApp : public ktxApp { " For ETC1S encoder '--encode etc1s' tunes codec parameters for \n" " better quality on normal maps (no selector RDO, no endpoint RDO).\n" " Only valid for linear textures.\n" - " --no_sse Forbid use of the SSE instruction set. Ignored if CPU does not\n" + " --no_sse\n" + " Forbid use of the SSE instruction set. Ignored if CPU does not\n" " support SSE. Only the Basis Universal compressor uses SSE.\n" " --bcmp\n" " Deprecated. Use '--encode etc1s' instead.\n" @@ -510,7 +583,8 @@ class scApp : public ktxApp { " By default, ETC1S / BasisLZ and ASTC compression will use the number of\n" " threads reported by thread::hardware_concurrency or 1 if value\n" " returned is 0.\n" - " --verbose Print encoder/compressor activity status to stdout. Currently\n" + " --verbose\n" + " Print encoder/compressor activity status to stdout. Currently\n" " only the astc, etc1s and uastc encoders emit status.\n" "\n"; ktxApp::usage(); From 49f793bdc50a175812c122c8a6fe2b5d113426a2 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Wed, 9 Jun 2021 17:08:50 +0100 Subject: [PATCH 25/65] Fix minor language --- tools/toktx/toktx.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/toktx/toktx.cc b/tools/toktx/toktx.cc index 40770325de..26e816d660 100644 --- a/tools/toktx/toktx.cc +++ b/tools/toktx/toktx.cc @@ -735,7 +735,7 @@ toktxApp::main(int argc, _TCHAR *argv[]) // If input is > 8bit and user wants LDR issue quality loss warning if (options.astc && image->getComponentSize() == 2 && options.astcopts.mode == KTX_PACK_ASTC_ENCODER_MODE_LDR) { - cerr << name << ": Warning! input file is 16bit but LDR option is provided." + cerr << name << ": Warning! input file is 16bit but LDR option is specified." << " Expect quality loss in the output." << endl; } @@ -743,7 +743,7 @@ toktxApp::main(int argc, _TCHAR *argv[]) // If input is 8bit and user wants HDR issue warning if (options.astc && image->getComponentSize() != 2 && options.astcopts.mode == KTX_PACK_ASTC_ENCODER_MODE_HDR) { - cerr << name << ": Warning! input file is not 16bit but HDR option is provided." + cerr << name << ": Warning! input file is not 16bit but HDR option is specified." << endl; } From 8860912e8134e6a6d2f027b4d1da2748f426ec45 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Thu, 10 Jun 2021 12:33:07 +0100 Subject: [PATCH 26/65] Add default options when no astc mode is specified --- include/ktx.h | 1 + lib/astc_encode.cpp | 2 +- tools/toktx/toktx.cc | 15 ++++++++++++--- utils/scapp.h | 2 +- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/include/ktx.h b/include/ktx.h index 647926bf14..dc25a4da4e 100644 --- a/include/ktx.h +++ b/include/ktx.h @@ -1020,6 +1020,7 @@ typedef enum ktx_pack_astc_encoder_function_e { * This and function is used later to derive the profile. */ typedef enum ktx_pack_astc_encoder_mode_e { + KTX_PACK_ASTC_ENCODER_MODE_DEFAULT, KTX_PACK_ASTC_ENCODER_MODE_LDR, KTX_PACK_ASTC_ENCODER_MODE_HDR, KTX_PACK_ASTC_ENCODER_MODE_MAX = KTX_PACK_ASTC_ENCODER_MODE_HDR diff --git a/lib/astc_encode.cpp b/lib/astc_encode.cpp index 444d5ceadb..2fcf86b9d3 100644 --- a/lib/astc_encode.cpp +++ b/lib/astc_encode.cpp @@ -309,7 +309,7 @@ astcEncoderMode(const char* mode) { else if (std::strcmp(mode, "hdr") == 0) return KTX_PACK_ASTC_ENCODER_MODE_HDR; - return KTX_PACK_ASTC_ENCODER_MODE_LDR; + return KTX_PACK_ASTC_ENCODER_MODE_DEFAULT; } /** diff --git a/tools/toktx/toktx.cc b/tools/toktx/toktx.cc index 26e816d660..f3280a9a1a 100644 --- a/tools/toktx/toktx.cc +++ b/tools/toktx/toktx.cc @@ -733,20 +733,29 @@ toktxApp::main(int argc, _TCHAR *argv[]) rescale); // If input is > 8bit and user wants LDR issue quality loss warning - if (options.astc && image->getComponentSize() == 2 + if (options.astc && image->getComponentSize() > 1 && options.astcopts.mode == KTX_PACK_ASTC_ENCODER_MODE_LDR) { cerr << name << ": Warning! input file is 16bit but LDR option is specified." << " Expect quality loss in the output." << endl; } - // If input is 8bit and user wants HDR issue warning - if (options.astc && image->getComponentSize() != 2 && + // If input is < 8bit and user wants HDR issue warning + if (options.astc && image->getComponentSize() <= 1 && options.astcopts.mode == KTX_PACK_ASTC_ENCODER_MODE_HDR) { cerr << name << ": Warning! input file is not 16bit but HDR option is specified." << endl; } + // If no astc mode option is specified and + // if input is <= 8bit default to LDR otherwise default to HDR + if (options.astc && options.astcopts.mode == KTX_PACK_ASTC_ENCODER_MODE_DEFAULT) { + if (image->getComponentSize() <= 1) + options.astcopts.mode = KTX_PACK_ASTC_ENCODER_MODE_LDR; + else + options.astcopts.mode = KTX_PACK_ASTC_ENCODER_MODE_HDR; + } + if (i == 0) { // First file. firstImageOETF = image->getOetf(); diff --git a/utils/scapp.h b/utils/scapp.h index 0d1101d3e6..64d3da092a 100644 --- a/utils/scapp.h +++ b/utils/scapp.h @@ -350,7 +350,7 @@ class scApp : public ktxApp { blockDimension.clear(); blockDimension = KTX_PACK_ASTC_BLOCK_DIMENSION_6x6; function.clear(); - // Default to unknown to have a chance to use to use color space from file + // Default to unknown to have a chance to use color space from file function = KTX_PACK_ASTC_ENCODER_FUNCTION_UNKNOWN; mode.clear(); qualityLevel.clear(); From a6e3138c40d1f88838b6ee16f6c76d09d8ce19d7 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Thu, 10 Jun 2021 12:38:01 +0100 Subject: [PATCH 27/65] Re-Add only 8bit format limiations untill full HDR support is added --- lib/astc_encode.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/astc_encode.cpp b/lib/astc_encode.cpp index 2fcf86b9d3..29f4d164f3 100644 --- a/lib/astc_encode.cpp +++ b/lib/astc_encode.cpp @@ -616,6 +616,9 @@ launchThreads(int threadCount, void (*func)(int, int, void*), void *payload) { * The texture image's format is a packed format * (e.g. RGB565). * @exception KTX_INVALID_OPERATION + * The texture image format's component size is not + * 8-bits. + * @exception KTX_INVALID_OPERATION * The texture's images are 1D. Only 2D images can * be supercompressed. * @exception KTX_INVALID_OPERATION @@ -653,6 +656,9 @@ ktxTexture_CompressAstcEx(ktxTexture* _This, ktxAstcParams* params) { uint32_t num_components, component_size; getDFDComponentInfoUnpacked(This->pDfd, &num_components, &component_size); + if (component_size != 1) + return KTX_INVALID_OPERATION; // Can only deal with 8-bit components at the moment + if (This->pData == NULL) { result = ktxTexture2_LoadImageData((ktxTexture2*)This, nullptr, 0); From 4c440bedafae7f45557caf231229bbb7f76c5264 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Fri, 11 Jun 2021 12:33:23 +0100 Subject: [PATCH 28/65] Add tests for astc support --- tests/testimages/astc_ldr_6x5.ktx2 | 3 +++ tests/testimages/astc_mipmap_ldr_4x4.ktx2 | 3 +++ tests/testimages/astc_mipmap_ldr_6x5.ktx2 | 3 +++ tests/testimages/astc_mipmap_ldr_6x6.ktx2 | 3 +++ tests/testimages/astc_mipmap_ldr_cubemap_10x12.ktx2 | 3 +++ tests/toktx-tests.cmake | 6 ++++++ 6 files changed, 21 insertions(+) create mode 100644 tests/testimages/astc_ldr_6x5.ktx2 create mode 100644 tests/testimages/astc_mipmap_ldr_4x4.ktx2 create mode 100644 tests/testimages/astc_mipmap_ldr_6x5.ktx2 create mode 100644 tests/testimages/astc_mipmap_ldr_6x6.ktx2 create mode 100644 tests/testimages/astc_mipmap_ldr_cubemap_10x12.ktx2 diff --git a/tests/testimages/astc_ldr_6x5.ktx2 b/tests/testimages/astc_ldr_6x5.ktx2 new file mode 100644 index 0000000000..0739b33cb3 --- /dev/null +++ b/tests/testimages/astc_ldr_6x5.ktx2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cccf65d11906ac0b38af3668097189ac1bcbcff5f27b6b80d317c01201805b2 +size 2243824 diff --git a/tests/testimages/astc_mipmap_ldr_4x4.ktx2 b/tests/testimages/astc_mipmap_ldr_4x4.ktx2 new file mode 100644 index 0000000000..342a2f4a8e --- /dev/null +++ b/tests/testimages/astc_mipmap_ldr_4x4.ktx2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31cc7b9f30793878a7e0d142aff9ed3e55500e19e970256a966cb9645c53cda8 +size 5593008 diff --git a/tests/testimages/astc_mipmap_ldr_6x5.ktx2 b/tests/testimages/astc_mipmap_ldr_6x5.ktx2 new file mode 100644 index 0000000000..d3bac39ad9 --- /dev/null +++ b/tests/testimages/astc_mipmap_ldr_6x5.ktx2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d23c6a7b4271c4e652385f5226d2bc9a56dfb68add874cd361298a81f2358050 +size 2994896 diff --git a/tests/testimages/astc_mipmap_ldr_6x6.ktx2 b/tests/testimages/astc_mipmap_ldr_6x6.ktx2 new file mode 100644 index 0000000000..601b4af06a --- /dev/null +++ b/tests/testimages/astc_mipmap_ldr_6x6.ktx2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c16a6d4043b6ad90731e5ed10033b85326bcf95bd95fca7cae9bb0e23434008a +size 2498288 diff --git a/tests/testimages/astc_mipmap_ldr_cubemap_10x12.ktx2 b/tests/testimages/astc_mipmap_ldr_cubemap_10x12.ktx2 new file mode 100644 index 0000000000..f610604670 --- /dev/null +++ b/tests/testimages/astc_mipmap_ldr_cubemap_10x12.ktx2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:019104bb355678a35e506b3a4fea84e112589926dbd53502acb9ef7cdb1dc16e +size 14986848 diff --git a/tests/toktx-tests.cmake b/tests/toktx-tests.cmake index f9672bb641..7ecb44bc80 100644 --- a/tests/toktx-tests.cmake +++ b/tests/toktx-tests.cmake @@ -168,3 +168,9 @@ set_tests_properties( PROPERTIES ENVIRONMENT TOKTX_OPTIONS=--lower_left_maps_to_s0t0 ) + +gencmpktx( astc_mipmap_ldr_cubemap_10x12 astc_mipmap_ldr_cubemap_10x12.ktx2 toktx.cubemap.filelist.txt "--encode astc --astc_blk_d 10x12 --genmipmap --cubemap" "" "../srcimages/Yokohama3/posx.jpg ../srcimages/Yokohama3/negx.jpg ../srcimages/Yokohama3/posy.jpg ../srcimages/Yokohama3/negy.jpg ../srcimages/Yokohama3/posz.jpg ../srcimages/Yokohama3/negz.jpg" ) +gencmpktx( astc_mipmap_ldr_4x4 astc_mipmap_ldr_4x4.ktx2 ../srcimages/Yokohama3/posx.jpg "--encode astc --astc_blk_d 4x4 --genmipmap" "" "" ) +gencmpktx( astc_ldr_6x5 astc_ldr_6x5.ktx2 ../srcimages/Yokohama3/posx.jpg "--encode astc --astc_blk_d 6x5" "" "" ) +gencmpktx( astc_mipmap_ldr_6x6 astc_mipmap_ldr_6x6.ktx2 ../srcimages/Yokohama3/posx.jpg "--encode astc --astc_blk_d 6x6 --genmipmap" "" "" ) +gencmpktx( astc_mipmap_ldr_6x5 astc_mipmap_ldr_6x5.ktx2 ../srcimages/Yokohama3/posx.jpg "--encode astc --astc_blk_d 6x5 --genmipmap" "" "" ) From e6fa2134c0c110ad81630413fc82c4f96e80a6d7 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Fri, 11 Jun 2021 16:17:38 +0100 Subject: [PATCH 29/65] Modify astc test images --- tests/testimages/astc_ldr_6x5.ktx2 | 4 ++-- tests/testimages/astc_mipmap_ldr_4x4.ktx2 | 4 ++-- tests/testimages/astc_mipmap_ldr_6x5.ktx2 | 4 ++-- tests/testimages/astc_mipmap_ldr_6x6.ktx2 | 4 ++-- tests/testimages/astc_mipmap_ldr_cubemap_10x12.ktx2 | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/testimages/astc_ldr_6x5.ktx2 b/tests/testimages/astc_ldr_6x5.ktx2 index 0739b33cb3..8c53f75754 100644 --- a/tests/testimages/astc_ldr_6x5.ktx2 +++ b/tests/testimages/astc_ldr_6x5.ktx2 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3cccf65d11906ac0b38af3668097189ac1bcbcff5f27b6b80d317c01201805b2 -size 2243824 +oid sha256:f262ceb5b9b1505e6637bd89dfc12e8c850ee56d3536f77f32a4d3c111fc2437 +size 2243808 diff --git a/tests/testimages/astc_mipmap_ldr_4x4.ktx2 b/tests/testimages/astc_mipmap_ldr_4x4.ktx2 index 342a2f4a8e..affe1fd52b 100644 --- a/tests/testimages/astc_mipmap_ldr_4x4.ktx2 +++ b/tests/testimages/astc_mipmap_ldr_4x4.ktx2 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:31cc7b9f30793878a7e0d142aff9ed3e55500e19e970256a966cb9645c53cda8 -size 5593008 +oid sha256:1baeecb49f4be0045b40ea350e75b1c66a9d7a4bf9a587221e6e78769ea85489 +size 5592976 diff --git a/tests/testimages/astc_mipmap_ldr_6x5.ktx2 b/tests/testimages/astc_mipmap_ldr_6x5.ktx2 index d3bac39ad9..8fc90a2a40 100644 --- a/tests/testimages/astc_mipmap_ldr_6x5.ktx2 +++ b/tests/testimages/astc_mipmap_ldr_6x5.ktx2 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d23c6a7b4271c4e652385f5226d2bc9a56dfb68add874cd361298a81f2358050 -size 2994896 +oid sha256:47b14d79e1fc3606afd543f7bd4de32421ec7885cab698c0cafca3e7efd450ce +size 2994864 diff --git a/tests/testimages/astc_mipmap_ldr_6x6.ktx2 b/tests/testimages/astc_mipmap_ldr_6x6.ktx2 index 601b4af06a..49b2b848fe 100644 --- a/tests/testimages/astc_mipmap_ldr_6x6.ktx2 +++ b/tests/testimages/astc_mipmap_ldr_6x6.ktx2 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c16a6d4043b6ad90731e5ed10033b85326bcf95bd95fca7cae9bb0e23434008a -size 2498288 +oid sha256:d412df3cd9d5a734a800195d49878c81a0d6be45ef7b84faa1c1dea733b0cdb9 +size 2498256 diff --git a/tests/testimages/astc_mipmap_ldr_cubemap_10x12.ktx2 b/tests/testimages/astc_mipmap_ldr_cubemap_10x12.ktx2 index f610604670..401062ef1e 100644 --- a/tests/testimages/astc_mipmap_ldr_cubemap_10x12.ktx2 +++ b/tests/testimages/astc_mipmap_ldr_cubemap_10x12.ktx2 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:019104bb355678a35e506b3a4fea84e112589926dbd53502acb9ef7cdb1dc16e -size 14986848 +oid sha256:e0e5ee19a5d5f3cd20dde1342a12b358187f2abe19a8d3728839ed6638edd53d +size 14986832 From 46e852227c8b496072db3dda6bdea6beebb9c949 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Fri, 11 Jun 2021 17:47:37 +0100 Subject: [PATCH 30/65] Add --test option to astc tests --- tests/toktx-tests.cmake | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/toktx-tests.cmake b/tests/toktx-tests.cmake index 7ecb44bc80..5abd4f117d 100644 --- a/tests/toktx-tests.cmake +++ b/tests/toktx-tests.cmake @@ -169,8 +169,8 @@ PROPERTIES ENVIRONMENT TOKTX_OPTIONS=--lower_left_maps_to_s0t0 ) -gencmpktx( astc_mipmap_ldr_cubemap_10x12 astc_mipmap_ldr_cubemap_10x12.ktx2 toktx.cubemap.filelist.txt "--encode astc --astc_blk_d 10x12 --genmipmap --cubemap" "" "../srcimages/Yokohama3/posx.jpg ../srcimages/Yokohama3/negx.jpg ../srcimages/Yokohama3/posy.jpg ../srcimages/Yokohama3/negy.jpg ../srcimages/Yokohama3/posz.jpg ../srcimages/Yokohama3/negz.jpg" ) -gencmpktx( astc_mipmap_ldr_4x4 astc_mipmap_ldr_4x4.ktx2 ../srcimages/Yokohama3/posx.jpg "--encode astc --astc_blk_d 4x4 --genmipmap" "" "" ) -gencmpktx( astc_ldr_6x5 astc_ldr_6x5.ktx2 ../srcimages/Yokohama3/posx.jpg "--encode astc --astc_blk_d 6x5" "" "" ) -gencmpktx( astc_mipmap_ldr_6x6 astc_mipmap_ldr_6x6.ktx2 ../srcimages/Yokohama3/posx.jpg "--encode astc --astc_blk_d 6x6 --genmipmap" "" "" ) -gencmpktx( astc_mipmap_ldr_6x5 astc_mipmap_ldr_6x5.ktx2 ../srcimages/Yokohama3/posx.jpg "--encode astc --astc_blk_d 6x5 --genmipmap" "" "" ) +gencmpktx( astc_mipmap_ldr_cubemap_10x12 astc_mipmap_ldr_cubemap_10x12.ktx2 toktx.cubemap.filelist.txt "--test --encode astc --astc_blk_d 10x12 --genmipmap --cubemap" "" "../srcimages/Yokohama3/posx.jpg ../srcimages/Yokohama3/negx.jpg ../srcimages/Yokohama3/posy.jpg ../srcimages/Yokohama3/negy.jpg ../srcimages/Yokohama3/posz.jpg ../srcimages/Yokohama3/negz.jpg" ) +gencmpktx( astc_mipmap_ldr_4x4 astc_mipmap_ldr_4x4.ktx2 ../srcimages/Yokohama3/posx.jpg "--test --encode astc --astc_blk_d 4x4 --genmipmap" "" "" ) +gencmpktx( astc_ldr_6x5 astc_ldr_6x5.ktx2 ../srcimages/Yokohama3/posx.jpg "--test --encode astc --astc_blk_d 6x5" "" "" ) +gencmpktx( astc_mipmap_ldr_6x6 astc_mipmap_ldr_6x6.ktx2 ../srcimages/Yokohama3/posx.jpg "--test --encode astc --astc_blk_d 6x6 --genmipmap" "" "" ) +gencmpktx( astc_mipmap_ldr_6x5 astc_mipmap_ldr_6x5.ktx2 ../srcimages/Yokohama3/posx.jpg "--test --encode astc --astc_blk_d 6x5 --genmipmap" "" "" ) From 56ecbdca6ed54954d28098c32d024a69b166443a Mon Sep 17 00:00:00 2001 From: wasimabbas-arm Date: Mon, 14 Jun 2021 15:40:54 +0100 Subject: [PATCH 31/65] Fix cpu architecture detection Fix cpu architecture detection --- CMakeLists.txt | 2 + cmake/cputypetest.cmake | 186 ++++++++++++++-------------------------- 2 files changed, 65 insertions(+), 123 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 720ad56064..8013f153c8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -663,6 +663,8 @@ else() set(ISA_AVX2 OFF) set(ASTCENC_POPCNT 0) set(ASTC_LIB_NAME astcenc-none-static) + else() + set(ARCH x64 CACHE STRING "Target architecture") endif() elseif(CPU_ARCHITECTURE STREQUAL arm OR CPU_ARCHITECTURE STREQUAL armv7 OR CPU_ARCHITECTURE STREQUAL armv8 OR CPU_ARCHITECTURE STREQUAL arm64) diff --git a/cmake/cputypetest.cmake b/cmake/cputypetest.cmake index 731cdcdad9..46a158ba1b 100644 --- a/cmake/cputypetest.cmake +++ b/cmake/cputypetest.cmake @@ -13,102 +13,59 @@ set(cputypetest_code " // // For CMake literal compatibility, this file must have no double quotes // -#ifdef _WIN32 - #ifdef _WIN64 - -#undef x86_64 -x86_64 - - #else - -#undef x86 -x86 - - #endif +#if defined _WIN32 +# if defined _WIN64 +# error ARCH_FOUND x86_64 +# else +# error ARCH_FOUND x86 +# endif #elif defined __APPLE__ - #include - #if TARGET_OS_IPHONE - #if TARGET_CPU_X86 - -#undef x86 -x86 - - #elif TARGET_CPU_X86_64 -#undef x86_64 -x86_64 - - #elif TARGET_CPU_ARM - -#undef armv7 -armv7 - - #elif TARGET_CPU_ARM64 - -#undef armv8 -armv8 - - #else - #error Unsupported cpu - #endif - #elif TARGET_OS_MAC - - #if defined __x86_64__ - -#undef x86_64 -x86_64 - - #elif defined __aarch64__ - -#undef arm64 -arm64 - - #else - #error Unsupported platform - #endif - - #else - #error Unsupported platform - #endif +# include +# if TARGET_OS_IPHONE +# if TARGET_CPU_X86 +# error ARCH_FOUND x86 +# elif TARGET_CPU_X86_64 +# error ARCH_FOUND x86_64 +# elif TARGET_CPU_ARM +# error ARCH_FOUND armv7 +# elif TARGET_CPU_ARM64 +# error ARCH_FOUND armv8 +# else +# error Unsupported cpu +# endif +# elif TARGET_OS_MAC +# if defined __x86_64__ +# error ARCH_FOUND x86_64 +# elif defined __aarch64__ +# error ARCH_FOUND arm64 +# else +# error Unsupported platform +# endif +# else +# error Unsupported platform +# endif #elif defined __linux - #ifdef __ANDROID__ - #ifdef __i386__ - -#undef x86 -x86 - - #elif defined __arm__ - -#undef armv7 -armv7 - - #elif defined __aarch64__ - -#undef armv8 -armv8 - - #else - #error Unsupported cpu - #endif - #else - #ifdef __LP64__ - -#undef x86_64 -x86_64 - - #else - -#undef x86 -x86 - - #endif - #endif +# ifdef __ANDROID__ +# ifdef __i386__ +# error ARCH_FOUND x86 +# elif defined __arm__ +# error ARCH_FOUND armv7 +# elif defined __aarch64__ +# error ARCH_FOUND armv8 +# else +# error Unsupported cpu +# endif +# else +# ifdef __LP64__ +# error ARCH_FOUND x86_64 +# else +# error ARCH_FOUND x86 +# endif +# endif #elif defined __EMSCRIPTEN__ - -#undef wasm -wasm - +# error ARCH_FOUND wasm #else - #error Unsupported cpu +# error Unsupported cpu #endif ") file(WRITE "${CMAKE_BINARY_DIR}/cputypetest.c" "${cputypetest_code}") @@ -124,26 +81,20 @@ function(set_target_processor_type out) set(${out} x86 PARENT_SCOPE) elseif(ANDROID_ABI AND "${ANDROID_ABI}" STREQUAL "x86_64") set(${out} x86_64 PARENT_SCOPE) - else() if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC") - set(C_PREPROCESS ${CMAKE_C_COMPILER} /EP /nologo) if("${CMAKE_GENERATOR_PLATFORM}" STREQUAL "ARM") set(processor "arm") elseif("${CMAKE_GENERATOR_PLATFORM}" STREQUAL "ARM64") set(processor "arm64") else() - set(C_PREPROCESS ${CMAKE_C_COMPILER} /EP /nologo) - execute_process( - COMMAND ${C_PREPROCESS} "${CMAKE_BINARY_DIR}/cputypetest.c" - OUTPUT_VARIABLE processor - OUTPUT_STRIP_TRAILING_WHITESPACE - # Specify this to block MSVC's output of the source file name - # so as not to trigger PowerShell's stop-on-error in CI. - # Unfortunately it suppresses all compile errors too hence - # the special case for MSVC. - ERROR_QUIET - ) + try_compile(res_var + ${CMAKE_BINARY_DIR}/CMakeTemp + ${CMAKE_BINARY_DIR}/cputypetest.c + OUTPUT_VARIABLE processor) + + string(REGEX MATCH "ARCH_FOUND ([_a-z0-9]+)" processor "${processor}") + string(REPLACE "ARCH_FOUND " "" processor "${processor}") endif() else() if(APPLE AND NOT "${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin") @@ -154,26 +105,15 @@ function(set_target_processor_type out) # type of ARM processor arbitrarily set armv8 for these systems. set(processor armv8) else() - if("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin") - if (CMAKE_OSX_SYSROOT) - set(TC_INCLUDE_DIR -I ${CMAKE_OSX_SYSROOT}/usr/include) - else() - # I have seen cases where CMAKE_OSX_SYSROOT is not defined - # for reasons I do not understand. Plus, uses can manually - # undefine it. This is the fallback. - set(TC_INCLUDE_DIR -I /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/include) - endif() - endif() - set(C_PREPROCESS ${CMAKE_C_COMPILER} ${TC_INCLUDE_DIR} -E -P) - execute_process( - COMMAND ${C_PREPROCESS} "${CMAKE_BINARY_DIR}/cputypetest.c" - OUTPUT_VARIABLE processor - OUTPUT_STRIP_TRAILING_WHITESPACE - ) + try_compile(res_var + ${CMAKE_BINARY_DIR}/CMakeTemp + ${CMAKE_BINARY_DIR}/cputypetest.c + OUTPUT_VARIABLE processor) + + string(REGEX MATCH "ARCH_FOUND ([_a-z0-9]+)" processor "${processor}") + string(REPLACE "ARCH_FOUND " "" processor "${processor}") endif() endif() - - string(STRIP "${processor}" processor) set(${out} ${processor} PARENT_SCOPE) endif() endfunction(set_target_processor_type) From 1dc4540fc693ad5cf8305f6141cfdd6f1d907c0b Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Mon, 21 Jun 2021 13:06:06 +0100 Subject: [PATCH 32/65] Add MSVC_RUNTIME_LIBRARY setting for ktx-software targets --- CMakeLists.txt | 5 +++++ cmake/cputypetest.cmake | 2 ++ 2 files changed, 7 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index b097b64754..a0376ca773 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,6 +3,11 @@ cmake_minimum_required(VERSION 3.15) +# Use a static runtime on MSVC builds (ignored on non-MSVC compilers) +set_property(TARGET ${NAME} + PROPERTY + MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>") + list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake/modules/") find_package(Bash REQUIRED) diff --git a/cmake/cputypetest.cmake b/cmake/cputypetest.cmake index 46a158ba1b..931ec30cde 100644 --- a/cmake/cputypetest.cmake +++ b/cmake/cputypetest.cmake @@ -93,6 +93,8 @@ function(set_target_processor_type out) ${CMAKE_BINARY_DIR}/cputypetest.c OUTPUT_VARIABLE processor) + # Inspired by answers to the following SO question + # https://stackoverflow.com/questions/39258250/how-to-detect-if-64-bit-msvc-with-cmake string(REGEX MATCH "ARCH_FOUND ([_a-z0-9]+)" processor "${processor}") string(REPLACE "ARCH_FOUND " "" processor "${processor}") endif() From f71b09eaa69ad8c90cb5a33dde067b5b9c795a6a Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Mon, 21 Jun 2021 14:28:31 +0100 Subject: [PATCH 33/65] Add MSVC_RUNTIME_LIBRARY setting for ktx-software targets --- CMakeLists.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a0376ca773..063638e222 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,11 +3,6 @@ cmake_minimum_required(VERSION 3.15) -# Use a static runtime on MSVC builds (ignored on non-MSVC compilers) -set_property(TARGET ${NAME} - PROPERTY - MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>") - list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake/modules/") find_package(Bash REQUIRED) @@ -689,6 +684,11 @@ target_link_libraries(ktx PRIVATE ${ASTC_LIB_NAME}) # Its a good optimisation anyways and most compilers now support it set_property(TARGET ktx PROPERTY INTERPROCEDURAL_OPTIMIZATION_RELEASE True) +# Use a static runtime on MSVC builds (ignored on non-MSVC compilers) +set_property(TARGET ktx + PROPERTY + MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>") + # Tools if(KTX_FEATURE_TOOLS) add_subdirectory(tools) From b7bc5a2cda6a8b292ed6673fbf57f33a6bfec59f Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Tue, 22 Jun 2021 12:47:49 +0100 Subject: [PATCH 34/65] Update cputypetest.cmake to master branch --- cmake/cputypetest.cmake | 197 +++++++++++++++++++++++++--------------- 1 file changed, 124 insertions(+), 73 deletions(-) diff --git a/cmake/cputypetest.cmake b/cmake/cputypetest.cmake index 931ec30cde..f334a248ce 100644 --- a/cmake/cputypetest.cmake +++ b/cmake/cputypetest.cmake @@ -13,61 +13,55 @@ set(cputypetest_code " // // For CMake literal compatibility, this file must have no double quotes // -#if defined _WIN32 -# if defined _WIN64 -# error ARCH_FOUND x86_64 -# else -# error ARCH_FOUND x86 -# endif -#elif defined __APPLE__ -# include -# if TARGET_OS_IPHONE -# if TARGET_CPU_X86 -# error ARCH_FOUND x86 -# elif TARGET_CPU_X86_64 -# error ARCH_FOUND x86_64 -# elif TARGET_CPU_ARM -# error ARCH_FOUND armv7 -# elif TARGET_CPU_ARM64 -# error ARCH_FOUND armv8 -# else -# error Unsupported cpu -# endif -# elif TARGET_OS_MAC -# if defined __x86_64__ -# error ARCH_FOUND x86_64 -# elif defined __aarch64__ -# error ARCH_FOUND arm64 -# else -# error Unsupported platform -# endif -# else -# error Unsupported platform -# endif -#elif defined __linux -# ifdef __ANDROID__ -# ifdef __i386__ -# error ARCH_FOUND x86 -# elif defined __arm__ -# error ARCH_FOUND armv7 -# elif defined __aarch64__ -# error ARCH_FOUND armv8 -# else -# error Unsupported cpu -# endif -# else -# ifdef __LP64__ -# error ARCH_FOUND x86_64 -# else -# error ARCH_FOUND x86 -# endif -# endif + +#if defined(__x86_64__) || defined(_M_X64) +x86_64 +#elif defined(i386) || defined(__i386__) || defined(__i386) || defined(_M_IX86) +x86_32 +#elif defined(__ARM_ARCH_2__) +armv2 +#elif defined(__ARM_ARCH_3__) || defined(__ARM_ARCH_3M__) +armv3 +#elif defined(__ARM_ARCH_4T__) || defined(__TARGET_ARM_4T) +armv4T +#elif defined(__ARM_ARCH_5_) || defined(__ARM_ARCH_5E_) +ARM5 +#elif defined(__ARM_ARCH_6T2_) || defined(__ARM_ARCH_6T2_) +armv6T2 +#elif defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) +armv6 +#elif defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) +armv7 +#elif defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) +armv7A +#elif defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) +armv7R +#elif defined(__ARM_ARCH_7M__) +armv7M +#elif defined(__ARM_ARCH_7S__) +armv7S +#elif defined(__aarch64__) || defined(_M_ARM64) +arm64 +#elif defined(mips) || defined(__mips__) || defined(__mips) +mips +#elif defined(__sh__) +superh +#elif defined(__powerpc) || defined(__powerpc__) || defined(__powerpc64__) || defined(__POWERPC__) || defined(__ppc__) || defined(__PPC__) || defined(_ARCH_PPC) +powerpc +#elif defined(__PPC64__) || defined(__ppc64__) || defined(_ARCH_PPC64) +powerpc64 +#elif defined(__sparc__) || defined(__sparc) +sparc +#elif defined(__m68k__) +m68k #elif defined __EMSCRIPTEN__ -# error ARCH_FOUND wasm +wasm #else -# error Unsupported cpu + #error Unsupported cpu #endif + ") + file(WRITE "${CMAKE_BINARY_DIR}/cputypetest.c" "${cputypetest_code}") cmake_policy(SET CMP0054 NEW) @@ -76,11 +70,12 @@ function(set_target_processor_type out) if(ANDROID_ABI AND "${ANDROID_ABI}" STREQUAL "armeabi-v7a") set(${out} armv7 PARENT_SCOPE) elseif(ANDROID_ABI AND "${ANDROID_ABI}" STREQUAL "arm64-v8a") - set(${out} armv8 PARENT_SCOPE) + set(${out} arm64 PARENT_SCOPE) elseif(ANDROID_ABI AND "${ANDROID_ABI}" STREQUAL "x86") set(${out} x86 PARENT_SCOPE) elseif(ANDROID_ABI AND "${ANDROID_ABI}" STREQUAL "x86_64") set(${out} x86_64 PARENT_SCOPE) + else() if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC") if("${CMAKE_GENERATOR_PLATFORM}" STREQUAL "ARM") @@ -88,34 +83,90 @@ function(set_target_processor_type out) elseif("${CMAKE_GENERATOR_PLATFORM}" STREQUAL "ARM64") set(processor "arm64") else() - try_compile(res_var - ${CMAKE_BINARY_DIR}/CMakeTemp - ${CMAKE_BINARY_DIR}/cputypetest.c - OUTPUT_VARIABLE processor) - - # Inspired by answers to the following SO question - # https://stackoverflow.com/questions/39258250/how-to-detect-if-64-bit-msvc-with-cmake - string(REGEX MATCH "ARCH_FOUND ([_a-z0-9]+)" processor "${processor}") - string(REPLACE "ARCH_FOUND " "" processor "${processor}") + set(C_PREPROCESS ${CMAKE_C_COMPILER} /EP /nologo) + # Versions of MSVC prior to VS2019 have a supporting dll which + # must be found along the search path. MSVC in VS 2019 just + # works. Whether due to not having this supporting dll or using + # a different way to locate it, is unknown. To make earlier + # versions work set the WORKING_DIR to the location of the + # support dll to ensure it is found. + string(REGEX REPLACE "/VC/.*$" "/Common7/IDE" + compiler_support_dir + ${CMAKE_C_COMPILER}) + execute_process( + COMMAND ${C_PREPROCESS} "${CMAKE_BINARY_DIR}/cputypetest.c" + WORKING_DIRECTORY ${compiler_support_dir} + OUTPUT_VARIABLE processor + OUTPUT_STRIP_TRAILING_WHITESPACE + # Specify this to block MSVC's output of the source file name + # so as not to trigger PowerShell's stop-on-error in CI. + # Unfortunately it suppresses all compile errors too hence + # the special case for MSVC. Which was convenient to have + # when we found the issue with earlier versions of VS. + ERROR_QUIET + ) endif() else() + # Apple's clang is a single compiler whose target is determined by + # its -arch or --target= options + # defaulting to the processor of the Mac it is running on. CMake does + # not set this directly when generating for Xcode. (I don't know what + # it does when generating makefiles. Therefore, in the Xcode case, + # compiling cputypetest.c with CMAKE_{C,CXX}_COMPILER results in + # processor being set to x86_86 or arm64 depending on the Mac it is + # running on. + # + # When building for iOS, only CMAKE_SYSTEN_NAME is specified during + # configuration. CMake leaves it up to Xcode or to the user passing + # a -sdk argument to xcbuild to make sure the correct + # is set when calling the compiler. + # + # When building for macOS (and possibly for iOS, not tested) the + # user can specify CMAKE_OSX_ARCHITECTURES if they want to compile + # for something different than the current processor. This could be + # set to $(ARCHS_STANDARD) an Xcode build setting whose value in + # recent Xcode versions is "x86_64 arm64" to create universal binaries. + # This is passed to Xcode which calls the compiler twice passing the + # same set of target_definitions each time so in this case we have + # to choose BASISU_SUPPORT_SSE=OFF or the arm64 compile will fail. + # + # The following code reflects all this. if(APPLE AND NOT "${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin") - # No other Apple systems are x64_64. When configuring for iOS, etc. - # CMAKE_C_COMPILER points at the HOST compiler - I can't find - # definitive documentation of what is supposed to happen - the - # test program above returns x86_64. Since we don't care what - # type of ARM processor arbitrarily set armv8 for these systems. + # Building for iOS, iPadOS, etc. Since we don't care what + # type of ARM processor, arbitrarily set armv8. + # It should be arm64 but there is a check in tests/CMakeLists.txt + # that is dropping loadtests for Apple Silicon arm64. set(processor armv8) + elseif(APPLE AND CMAKE_OSX_ARCHITECTURES) + string(STRIP "${CMAKE_OSX_ARCHITECTURES}" architectures) + if("${architectures}" STREQUAL "$(ARCHS_STANDARD)") + # Choose arm64 so SSE support is disabled. + set(processor arm64) + elseif("${architectures}" MATCHES "[^ ]+[ ]+[^ ]+") + # Multiple words, choose arm64 so SSE support is disabled. + set(processor arm64) + elseif(APPLE AND "${CMAKE_OSX_ARCHITECTURES}" STREQUAL "x86_64") + set(processor x86_64) + elseif(APPLE AND "${CMAKE_OSX_ARCHITECTURES}" STREQUAL "arm64") + set(processor arm64) + else() + # Choose arm64 so SSE support is disabled. + set(processor arm64) + endif() + unset(architectures) else() - try_compile(res_var - ${CMAKE_BINARY_DIR}/CMakeTemp - ${CMAKE_BINARY_DIR}/cputypetest.c - OUTPUT_VARIABLE processor) - - string(REGEX MATCH "ARCH_FOUND ([_a-z0-9]+)" processor "${processor}") - string(REPLACE "ARCH_FOUND " "" processor "${processor}") + # This will distinguish between M1 and Intel Macs + set(C_PREPROCESS ${CMAKE_C_COMPILER} -E -P) + execute_process( + COMMAND ${C_PREPROCESS} "${CMAKE_BINARY_DIR}/cputypetest.c" + OUTPUT_VARIABLE processor + OUTPUT_STRIP_TRAILING_WHITESPACE + ) endif() endif() + + string(STRIP "${processor}" processor) + #message(STATUS "*** set_target_processor_type ***: processor is ${processor}") set(${out} ${processor} PARENT_SCOPE) endif() endfunction(set_target_processor_type) From 31023ed8c436e6f62cf69bdf41925575f3712de1 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Fri, 25 Jun 2021 16:17:04 +0100 Subject: [PATCH 35/65] Update astc encoder build options --- CMakeLists.txt | 16 +++++----------- ci_scripts/build_android.sh | 6 ++++-- ci_scripts/build_android_debug.sh | 6 ++++-- ci_scripts/build_macos.sh | 4 ++-- 4 files changed, 15 insertions(+), 17 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6177219948..6f0d78d5f8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -641,11 +641,10 @@ add_subdirectory(interface/basisu_c_binding) # Only one architecture is supported at once, if neither of # ISA_SSE41 and ISA_SSE2 are defined ISA_AVX2 is chosen. -# If ISA_AVX2 fails to compile user must chose other x86 options +# If ISA_AVX2 fails to compile user must chose other x86 options. # On arm based systems ISA_NEON is default if (${ISA_NONE}) - set(ISA_NONE ON) set(ASTC_LIB_NAME astcenc-none-static) else() if(CPU_ARCHITECTURE STREQUAL x86_64 OR CPU_ARCHITECTURE STREQUAL x86) @@ -662,17 +661,12 @@ else() set(ISA_AVX2 OFF) set(ASTCENC_POPCNT 0) set(ASTC_LIB_NAME astcenc-none-static) - else() - set(ARCH x64 CACHE STRING "Target architecture") endif() - elseif(CPU_ARCHITECTURE STREQUAL arm OR CPU_ARCHITECTURE STREQUAL armv7 - OR CPU_ARCHITECTURE STREQUAL armv8 OR CPU_ARCHITECTURE STREQUAL arm64) - set(ISA_NEON ON) + elseif(CPU_ARCHITECTURE STREQUAL armv8 OR CPU_ARCHITECTURE STREQUAL arm64) set(ASTC_LIB_NAME astcenc-neon-static) - set(ARCH aarch64 CACHE STRING "Target architecture") - if(CPU_ARCHITECTURE STREQUAL armv7) - set(ARCH aarch32 CACHE STRING "Target architecture" FORCE) - endif() + else() + message(STATUS "Unsupported ISA for ${CPU_ARCHITECTURE} arch, using ISA_NONE.") + set(ISA_NONE ON) endif() endif() diff --git a/ci_scripts/build_android.sh b/ci_scripts/build_android.sh index 42408cb544..877a2838cb 100755 --- a/ci_scripts/build_android.sh +++ b/ci_scripts/build_android.sh @@ -6,9 +6,10 @@ set -e # Fallback to arm64-v8a ANDROID_ABI=${ANDROID_ABI:-'arm64-v8a'} +ASTC_ISA=${ASTC_ISA:-'ISA_NONE=ON'} # You need to set the following environment variables first -# ANDROID_NDK= +# ANDROID_NDK= echo "Configure KTX-Software (Android $ANDROID_ABI Release)" cmake . -G Ninja -B "build-android-$ANDROID_ABI" \ @@ -17,7 +18,8 @@ cmake . -G Ninja -B "build-android-$ANDROID_ABI" \ -DANDROID_NDK="$ANDROID_NDK" \ -DCMAKE_TOOLCHAIN_FILE="$ANDROID_NDK/build/cmake/android.toolchain.cmake" \ -DCMAKE_BUILD_TYPE=Release \ --DBASISU_SUPPORT_SSE=OFF +-DBASISU_SUPPORT_SSE=OFF \ +-D${ASTC_ISA} pushd "build-android-$ANDROID_ABI" diff --git a/ci_scripts/build_android_debug.sh b/ci_scripts/build_android_debug.sh index dfa79e8fc6..06e550ae5d 100755 --- a/ci_scripts/build_android_debug.sh +++ b/ci_scripts/build_android_debug.sh @@ -6,9 +6,10 @@ set -e # Fallback to arm64-v8a ANDROID_ABI=${ANDROID_ABI:-'arm64-v8a'} +ASTC_ISA=${ASTC_ISA:-'ISA_NONE=ON'} # You need to set the following environment variables first -# ANDROID_NDK= +# ANDROID_NDK= echo "Configure KTX-Software (Android $ANDROID_ABI Debug)" cmake . -G Ninja -B "build-android-$ANDROID_ABI-debug" \ @@ -17,7 +18,8 @@ cmake . -G Ninja -B "build-android-$ANDROID_ABI-debug" \ -DANDROID_NDK="$ANDROID_NDK" \ -DCMAKE_TOOLCHAIN_FILE="$ANDROID_NDK/build/cmake/android.toolchain.cmake" \ -DCMAKE_BUILD_TYPE=Debug \ --DBASISU_SUPPORT_SSE=OFF +-DBASISU_SUPPORT_SSE=OFF \ +-D${ASTC_ISA} pushd "build-android-$ANDROID_ABI-debug" diff --git a/ci_scripts/build_macos.sh b/ci_scripts/build_macos.sh index 46e3d98197..7e7351de9e 100755 --- a/ci_scripts/build_macos.sh +++ b/ci_scripts/build_macos.sh @@ -37,7 +37,7 @@ else # No secure variables means a PR or fork build. fi echo "Configure KTX-Software (macOS) without SSE support" -cmake -GXcode -Bbuild-macos-nosse -DBASISU_SUPPORT_SSE=OFF +cmake -GXcode -Bbuild-macos-nosse -DBASISU_SUPPORT_SSE=OFF -DISA_NONE=ON # Cause the build pipes below to set the exit to the exit code of the # last program to exit non-zero. @@ -91,7 +91,7 @@ popd echo "Configure KTX-Software (iOS)" -cmake -GXcode -Bbuild-ios -DARCH=aarch64 -DCMAKE_SYSTEM_NAME=iOS -DKTX_FEATURE_LOADTEST_APPS=ON -DKTX_FEATURE_DOC=ON +cmake -GXcode -Bbuild-ios -DISA_NEON=ON -DCMAKE_SYSTEM_NAME=iOS -DKTX_FEATURE_LOADTEST_APPS=ON -DKTX_FEATURE_DOC=ON pushd build-ios echo "Build KTX-Software (iOS Debug)" cmake --build . --config Debug -- -sdk iphoneos CODE_SIGN_IDENTITY="" CODE_SIGNING_ALLOWED=NO CODE_SIGNING_REQUIRED=NO | tee -a fullbuild.log | xcpretty From 52da83aff261a0689ba885229c665ee38d3cfd72 Mon Sep 17 00:00:00 2001 From: wasimabbas-arm Date: Fri, 25 Jun 2021 16:19:59 +0100 Subject: [PATCH 36/65] Update main.yml --- .github/workflows/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 50facbbf6f..1753129e33 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -39,12 +39,14 @@ jobs: env: ANDROID_ABI: arm64-v8a ANDROID_NDK: ${{ steps.setup-ndk.outputs.ndk-path }} + ASTC_ISA: "ISA_NEON=ON" - name: android_arm64-v8a run: ./ci_scripts/build_android.sh env: ANDROID_ABI: arm64-v8a ANDROID_NDK: ${{ steps.setup-ndk.outputs.ndk-path }} + ASTC_ISA: "ISA_NEON=ON" - name: android_armeabi-v7a run: ./ci_scripts/build_android.sh From ae193dc46d0dd0e2cdeffe8d897864e8393d9de0 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Fri, 25 Jun 2021 16:31:41 +0100 Subject: [PATCH 37/65] Reset SSE support for mac os --- ci_scripts/build_macos.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci_scripts/build_macos.sh b/ci_scripts/build_macos.sh index a531bafdd4..e90a1074db 100755 --- a/ci_scripts/build_macos.sh +++ b/ci_scripts/build_macos.sh @@ -48,7 +48,7 @@ fi echo "Configure KTX-Software (macOS x86_64) with SSE support" cmake -GXcode -Bbuild-macos-sse \ -DCMAKE_OSX_ARCHITECTURES="x86_64" \ - -DBASISU_SUPPORT_SSE=ON -DISA_NONE-ON + -DBASISU_SUPPORT_SSE=ON # Cause the build pipes below to set the exit to the exit code of the # last program to exit non-zero. From aff59edcf922f1c6758012793beecd2c803447a7 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Mon, 28 Jun 2021 10:01:09 +0100 Subject: [PATCH 38/65] Fix link errors for Visual studio 16 --- lib/astc_encode.cpp | 90 --------------------------------------------- utils/scapp.h | 88 ++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 85 insertions(+), 93 deletions(-) diff --git a/lib/astc_encode.cpp b/lib/astc_encode.cpp index 29f4d164f3..0a7e08e786 100644 --- a/lib/astc_encode.cpp +++ b/lib/astc_encode.cpp @@ -222,96 +222,6 @@ astcDefaultOptions() { return params; } -/** - * @memberof ktxTexture - * @ingroup write - * @~English - * @brief Creates valid ASTC block dimension from string. - * - * @return Valid ktx_pack_astc_block_dimension_e from string - */ -ktx_pack_astc_block_dimension_e -astcBlockDimension(const char* block_size) { - static std::unordered_map - astc_blocks_mapping{{"4x4", KTX_PACK_ASTC_BLOCK_DIMENSION_4x4}, - {"5x4", KTX_PACK_ASTC_BLOCK_DIMENSION_5x4}, - {"5x5", KTX_PACK_ASTC_BLOCK_DIMENSION_5x5}, - {"6x5", KTX_PACK_ASTC_BLOCK_DIMENSION_6x5}, - {"6x6", KTX_PACK_ASTC_BLOCK_DIMENSION_6x6}, - {"8x5", KTX_PACK_ASTC_BLOCK_DIMENSION_8x5}, - {"8x6", KTX_PACK_ASTC_BLOCK_DIMENSION_8x6}, - {"10x5", KTX_PACK_ASTC_BLOCK_DIMENSION_10x5}, - {"10x6", KTX_PACK_ASTC_BLOCK_DIMENSION_10x6}, - {"8x8", KTX_PACK_ASTC_BLOCK_DIMENSION_8x8}, - {"10x8", KTX_PACK_ASTC_BLOCK_DIMENSION_10x8}, - {"10x10", KTX_PACK_ASTC_BLOCK_DIMENSION_10x10}, - {"12x10", KTX_PACK_ASTC_BLOCK_DIMENSION_12x10}, - {"12x12", KTX_PACK_ASTC_BLOCK_DIMENSION_12x12}, - {"3x3x3", KTX_PACK_ASTC_BLOCK_DIMENSION_3x3x3}, - {"4x3x3", KTX_PACK_ASTC_BLOCK_DIMENSION_4x3x3}, - {"4x4x3", KTX_PACK_ASTC_BLOCK_DIMENSION_4x4x3}, - {"4x4x4", KTX_PACK_ASTC_BLOCK_DIMENSION_4x4x4}, - {"5x4x4", KTX_PACK_ASTC_BLOCK_DIMENSION_5x4x4}, - {"5x5x4", KTX_PACK_ASTC_BLOCK_DIMENSION_5x5x4}, - {"5x5x5", KTX_PACK_ASTC_BLOCK_DIMENSION_5x5x5}, - {"6x5x5", KTX_PACK_ASTC_BLOCK_DIMENSION_6x5x5}, - {"6x6x5", KTX_PACK_ASTC_BLOCK_DIMENSION_6x6x5}, - {"6x6x6", KTX_PACK_ASTC_BLOCK_DIMENSION_6x6x6}}; - - auto opt = astc_blocks_mapping.find(block_size); - - if (opt != astc_blocks_mapping.end()) - return opt->second; - - return KTX_PACK_ASTC_BLOCK_DIMENSION_6x6; -} - -/** - * @memberof ktxTexture - * @ingroup write - * @~English - * @brief Creates valid ASTC quality from string. - * - * @return Valid ktx_pack_astc_quality_e from string - */ -ktx_pack_astc_quality_levels_e -astcQualityLevel(const char *quality) { - - static std::unordered_map astc_quality_mapping{ - {"fastest", KTX_PACK_ASTC_QUALITY_LEVEL_FASTEST}, - {"fast", KTX_PACK_ASTC_QUALITY_LEVEL_FAST}, - {"medium", KTX_PACK_ASTC_QUALITY_LEVEL_MEDIUM}, - {"thorough", KTX_PACK_ASTC_QUALITY_LEVEL_THOROUGH}, - {"exhaustive", KTX_PACK_ASTC_QUALITY_LEVEL_EXHAUSTIVE} - }; - - auto opt = astc_quality_mapping.find(quality); - - if (opt != astc_quality_mapping.end()) - return opt->second; - - return KTX_PACK_ASTC_QUALITY_LEVEL_MEDIUM; -} - -/** - * @memberof ktxTexture - * @ingroup write - * @~English - * @brief Creates valid ASTC mode from string. - * - * @return Valid ktx_pack_astc_mode_e from string - */ -ktx_pack_astc_encoder_mode_e -astcEncoderMode(const char* mode) { - if (std::strcmp(mode, "ldr") == 0) - return KTX_PACK_ASTC_ENCODER_MODE_LDR; - else if (std::strcmp(mode, "hdr") == 0) - return KTX_PACK_ASTC_ENCODER_MODE_HDR; - - return KTX_PACK_ASTC_ENCODER_MODE_DEFAULT; -} - /** * @memberof ktxTexture * @ingroup write diff --git a/utils/scapp.h b/utils/scapp.h index 64d3da092a..1a8324cb6d 100644 --- a/utils/scapp.h +++ b/utils/scapp.h @@ -6,6 +6,7 @@ #include #include "ktxapp.h" +#include template struct clampedOption @@ -38,14 +39,95 @@ struct clampedOption T max; }; +/** + * @memberof ktxTexture + * @ingroup write + * @~English + * @brief Creates valid ASTC block dimension from string. + * + * @return Valid ktx_pack_astc_block_dimension_e from string + */ ktx_pack_astc_block_dimension_e -astcBlockDimension(const char* block_dimension); +astcBlockDimension(const char* block_size) { + static std::unordered_map + astc_blocks_mapping{{"4x4", KTX_PACK_ASTC_BLOCK_DIMENSION_4x4}, + {"5x4", KTX_PACK_ASTC_BLOCK_DIMENSION_5x4}, + {"5x5", KTX_PACK_ASTC_BLOCK_DIMENSION_5x5}, + {"6x5", KTX_PACK_ASTC_BLOCK_DIMENSION_6x5}, + {"6x6", KTX_PACK_ASTC_BLOCK_DIMENSION_6x6}, + {"8x5", KTX_PACK_ASTC_BLOCK_DIMENSION_8x5}, + {"8x6", KTX_PACK_ASTC_BLOCK_DIMENSION_8x6}, + {"10x5", KTX_PACK_ASTC_BLOCK_DIMENSION_10x5}, + {"10x6", KTX_PACK_ASTC_BLOCK_DIMENSION_10x6}, + {"8x8", KTX_PACK_ASTC_BLOCK_DIMENSION_8x8}, + {"10x8", KTX_PACK_ASTC_BLOCK_DIMENSION_10x8}, + {"10x10", KTX_PACK_ASTC_BLOCK_DIMENSION_10x10}, + {"12x10", KTX_PACK_ASTC_BLOCK_DIMENSION_12x10}, + {"12x12", KTX_PACK_ASTC_BLOCK_DIMENSION_12x12}, + {"3x3x3", KTX_PACK_ASTC_BLOCK_DIMENSION_3x3x3}, + {"4x3x3", KTX_PACK_ASTC_BLOCK_DIMENSION_4x3x3}, + {"4x4x3", KTX_PACK_ASTC_BLOCK_DIMENSION_4x4x3}, + {"4x4x4", KTX_PACK_ASTC_BLOCK_DIMENSION_4x4x4}, + {"5x4x4", KTX_PACK_ASTC_BLOCK_DIMENSION_5x4x4}, + {"5x5x4", KTX_PACK_ASTC_BLOCK_DIMENSION_5x5x4}, + {"5x5x5", KTX_PACK_ASTC_BLOCK_DIMENSION_5x5x5}, + {"6x5x5", KTX_PACK_ASTC_BLOCK_DIMENSION_6x5x5}, + {"6x6x5", KTX_PACK_ASTC_BLOCK_DIMENSION_6x6x5}, + {"6x6x6", KTX_PACK_ASTC_BLOCK_DIMENSION_6x6x6}}; + + auto opt = astc_blocks_mapping.find(block_size); + + if (opt != astc_blocks_mapping.end()) + return opt->second; + + return KTX_PACK_ASTC_BLOCK_DIMENSION_6x6; +} +/** + * @memberof ktxTexture + * @ingroup write + * @~English + * @brief Creates valid ASTC quality from string. + * + * @return Valid ktx_pack_astc_quality_e from string + */ ktx_pack_astc_quality_levels_e -astcQualityLevel(const char* quality); +astcQualityLevel(const char *quality) { + + static std::unordered_map astc_quality_mapping{ + {"fastest", KTX_PACK_ASTC_QUALITY_LEVEL_FASTEST}, + {"fast", KTX_PACK_ASTC_QUALITY_LEVEL_FAST}, + {"medium", KTX_PACK_ASTC_QUALITY_LEVEL_MEDIUM}, + {"thorough", KTX_PACK_ASTC_QUALITY_LEVEL_THOROUGH}, + {"exhaustive", KTX_PACK_ASTC_QUALITY_LEVEL_EXHAUSTIVE} + }; + + auto opt = astc_quality_mapping.find(quality); + + if (opt != astc_quality_mapping.end()) + return opt->second; + return KTX_PACK_ASTC_QUALITY_LEVEL_MEDIUM; +} + +/** + * @memberof ktxTexture + * @ingroup write + * @~English + * @brief Creates valid ASTC mode from string. + * + * @return Valid ktx_pack_astc_mode_e from string + */ ktx_pack_astc_encoder_mode_e -astcEncoderMode(const char* mode); +astcEncoderMode(const char* mode) { + if (std::strcmp(mode, "ldr") == 0) + return KTX_PACK_ASTC_ENCODER_MODE_LDR; + else if (std::strcmp(mode, "hdr") == 0) + return KTX_PACK_ASTC_ENCODER_MODE_HDR; + + return KTX_PACK_ASTC_ENCODER_MODE_DEFAULT; +} /* // Markdown doesn't work in files included by snipped{doc} or include{doc} From b382950903622a1f557187f76457bbf15654f768 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Mon, 28 Jun 2021 10:15:15 +0100 Subject: [PATCH 39/65] Add max replacement for Visual studio --- utils/scapp.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utils/scapp.h b/utils/scapp.h index 1a8324cb6d..124e5646e1 100644 --- a/utils/scapp.h +++ b/utils/scapp.h @@ -6,6 +6,7 @@ #include #include "ktxapp.h" +#include "ktxint.h" #include template @@ -451,7 +452,7 @@ class scApp : public ktxApp { commandOptions() : zcmpLevel(ZSTD_CLEVEL_DEFAULT, 1U, 22U), - threadCount(max(1U, thread::hardware_concurrency()) , 1U, 10000U) + threadCount(MAX(1U, thread::hardware_concurrency()) , 1U, 10000U) { ktx2 = false; etc1s = false; From d775d0676e8d11f0f36ffef1ecb5c7c19c6b6b9e Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Mon, 28 Jun 2021 11:41:42 +0100 Subject: [PATCH 40/65] Linux ci fixes --- tests/testimages/astc_ldr_6x5.ktx2 | 2 +- tests/testimages/astc_mipmap_ldr_4x4.ktx2 | 2 +- tests/testimages/astc_mipmap_ldr_6x5.ktx2 | 2 +- tests/testimages/astc_mipmap_ldr_6x6.ktx2 | 2 +- utils/scapp.h | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/testimages/astc_ldr_6x5.ktx2 b/tests/testimages/astc_ldr_6x5.ktx2 index 8c53f75754..78cd407f5c 100644 --- a/tests/testimages/astc_ldr_6x5.ktx2 +++ b/tests/testimages/astc_ldr_6x5.ktx2 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f262ceb5b9b1505e6637bd89dfc12e8c850ee56d3536f77f32a4d3c111fc2437 +oid sha256:1f9b8cc6f41c083286e3a97c3f90f29c39fba71fa732a442b2197cf40dad6134 size 2243808 diff --git a/tests/testimages/astc_mipmap_ldr_4x4.ktx2 b/tests/testimages/astc_mipmap_ldr_4x4.ktx2 index affe1fd52b..4ec17d2d6c 100644 --- a/tests/testimages/astc_mipmap_ldr_4x4.ktx2 +++ b/tests/testimages/astc_mipmap_ldr_4x4.ktx2 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1baeecb49f4be0045b40ea350e75b1c66a9d7a4bf9a587221e6e78769ea85489 +oid sha256:83ad258f62bf929b0b9bc36d3e19d39638b5e709ef0772bf5e08b00792b3b9e7 size 5592976 diff --git a/tests/testimages/astc_mipmap_ldr_6x5.ktx2 b/tests/testimages/astc_mipmap_ldr_6x5.ktx2 index 8fc90a2a40..6a3358b15e 100644 --- a/tests/testimages/astc_mipmap_ldr_6x5.ktx2 +++ b/tests/testimages/astc_mipmap_ldr_6x5.ktx2 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:47b14d79e1fc3606afd543f7bd4de32421ec7885cab698c0cafca3e7efd450ce +oid sha256:c97ed3cedd7db7f9a5e4e8a359dbce73b14d7bf0b3d67ccd92c022ae13382a14 size 2994864 diff --git a/tests/testimages/astc_mipmap_ldr_6x6.ktx2 b/tests/testimages/astc_mipmap_ldr_6x6.ktx2 index 49b2b848fe..0dc1f33b53 100644 --- a/tests/testimages/astc_mipmap_ldr_6x6.ktx2 +++ b/tests/testimages/astc_mipmap_ldr_6x6.ktx2 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d412df3cd9d5a734a800195d49878c81a0d6be45ef7b84faa1c1dea733b0cdb9 +oid sha256:fed44dfe8a2f46364bb5d59e748feee6e85825ff3c357cb219ed972b720052e2 size 2498256 diff --git a/utils/scapp.h b/utils/scapp.h index 124e5646e1..f8bd8f955c 100644 --- a/utils/scapp.h +++ b/utils/scapp.h @@ -122,9 +122,9 @@ astcQualityLevel(const char *quality) { */ ktx_pack_astc_encoder_mode_e astcEncoderMode(const char* mode) { - if (std::strcmp(mode, "ldr") == 0) + if (strcmp(mode, "ldr") == 0) return KTX_PACK_ASTC_ENCODER_MODE_LDR; - else if (std::strcmp(mode, "hdr") == 0) + else if (strcmp(mode, "hdr") == 0) return KTX_PACK_ASTC_ENCODER_MODE_HDR; return KTX_PACK_ASTC_ENCODER_MODE_DEFAULT; From 18abe5f5db45c3d68c70aac16e07e6c0c4a2de96 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Mon, 28 Jun 2021 11:53:13 +0100 Subject: [PATCH 41/65] Documentation fixes --- utils/scapp.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/utils/scapp.h b/utils/scapp.h index f8bd8f955c..78b155ac8a 100644 --- a/utils/scapp.h +++ b/utils/scapp.h @@ -204,11 +204,11 @@ astcEncoderMode(const char* mode) {
--no_multithreading
Disable multithreading. Deprecated. For backward compatibility. Use @b --threads 1 instead.
-
--clevel
+
--clevel <level>
ETC1S / BasisLZ compression level, an encoding speed vs. quality tradeoff. Range is [0,5], default is 1. Higher values are slower, but give higher quality.
-
--qlevel
+
--qlevel <level>
ETC1S / BasisLZ quality level. Range is [1,255]. Lower gives better compression/lower quality/faster. Higher gives less compression/higher quality/slower. @b --qlevel automatically @@ -228,14 +228,14 @@ astcEncoderMode(const char* mode) {
--max_endpoints <arg>
Manually set the maximum number of color endpoint clusters. Range is [1,16128]. Default is 0, unset.
-
--endpoint_rdo_threshold
+
--endpoint_rdo_threshold <arg>
Set endpoint RDO quality threshold. The default is 1.25. Lower is higher quality but less quality per output bit (try [1.0,3.0]). This will override the value chosen by @b --qlevel.
--max_selectors <arg>
Manually set the maximum number of color selector clusters from [1,16128]. Default is 0, unset.
-
--selector_rdo_threshold
+
--selector_rdo_threshold <arg>
Set selector RDO quality threshold. The default is 1.25. Lower is higher quality but less quality per output bit (try [1.0,3.0]). This will override the value chosen by @b --qlevel.
From 019cbb4d6904bf9e56f6a81948c4509906b57069 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Mon, 28 Jun 2021 12:46:03 +0100 Subject: [PATCH 42/65] Fix dfd.h include --- lib/astc_encode.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/astc_encode.cpp b/lib/astc_encode.cpp index 0a7e08e786..429856db61 100644 --- a/lib/astc_encode.cpp +++ b/lib/astc_encode.cpp @@ -24,7 +24,7 @@ #include #include -#include "dfd.h" +#include "dfdutils/dfd.h" #include "ktx.h" #include "ktxint.h" #include "texture2.h" From bd69070759c69950811884150de5eec263cc344f Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Mon, 28 Jun 2021 13:25:26 +0100 Subject: [PATCH 43/65] Fix emscription include issue --- lib/astc_encode.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/astc_encode.cpp b/lib/astc_encode.cpp index 429856db61..e482bb986d 100644 --- a/lib/astc_encode.cpp +++ b/lib/astc_encode.cpp @@ -31,7 +31,7 @@ #include "vkformat_enum.h" #include "vk_format.h" -#include "astcenc.h" +#include "astc-encoder/Source/astcenc.h" #include "../tools/toktx/image.hpp" // Provide pthreads support on windows From e2125c5aa7edc6c6eb4ea8b2c0c121f77203d669 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Mon, 28 Jun 2021 13:26:57 +0100 Subject: [PATCH 44/65] Fix universal build for mac os --- CMakeLists.txt | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6f0d78d5f8..bc782ac032 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -644,7 +644,17 @@ add_subdirectory(interface/basisu_c_binding) # If ISA_AVX2 fails to compile user must chose other x86 options. # On arm based systems ISA_NEON is default -if (${ISA_NONE}) +list(FIND CMAKE_OSX_ARCHITECTURES "$(ARCHS_STANDARD)" ASTC_BUILD_UNIVERSAL) + +if(${ASTC_BUILD_UNIVERSAL} EQUAL -1) + set(ASTC_BUILD_UNIVERSAL OFF) +else() + set(ASTC_BUILD_UNIVERSAL ON) +endif() + +if (${ASTC_BUILD_UNIVERSAL}) + set(ASTC_LIB_NAME astcenc-static) +elseif (${ISA_NONE}) set(ASTC_LIB_NAME astcenc-none-static) else() if(CPU_ARCHITECTURE STREQUAL x86_64 OR CPU_ARCHITECTURE STREQUAL x86) From 65286c8d2a0947d8c3e4cc4ca506bcdd12b56c1b Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Mon, 28 Jun 2021 16:56:57 +0100 Subject: [PATCH 45/65] Simply universal build option --- CMakeLists.txt | 50 +++++++++++++++++++++++--------------------------- 1 file changed, 23 insertions(+), 27 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bc782ac032..37008819a3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -647,37 +647,33 @@ add_subdirectory(interface/basisu_c_binding) list(FIND CMAKE_OSX_ARCHITECTURES "$(ARCHS_STANDARD)" ASTC_BUILD_UNIVERSAL) if(${ASTC_BUILD_UNIVERSAL} EQUAL -1) - set(ASTC_BUILD_UNIVERSAL OFF) -else() - set(ASTC_BUILD_UNIVERSAL ON) -endif() - -if (${ASTC_BUILD_UNIVERSAL}) - set(ASTC_LIB_NAME astcenc-static) -elseif (${ISA_NONE}) - set(ASTC_LIB_NAME astcenc-none-static) -else() - if(CPU_ARCHITECTURE STREQUAL x86_64 OR CPU_ARCHITECTURE STREQUAL x86) - if (${ISA_SSE41}) - set(ASTC_LIB_NAME astcenc-sse4.1-static) - elseif (${ISA_SSE2}) - set(ASTC_LIB_NAME astcenc-sse2-static) + if (${ISA_NONE}) + set(ASTC_LIB_NAME astcenc-none-static) + else() + if(CPU_ARCHITECTURE STREQUAL x86_64 OR CPU_ARCHITECTURE STREQUAL x86) + if (${ISA_SSE41}) + set(ASTC_LIB_NAME astcenc-sse4.1-static) + elseif (${ISA_SSE2}) + set(ASTC_LIB_NAME astcenc-sse2-static) + else() + set(ISA_AVX2 ON) + set(ASTC_LIB_NAME astcenc-avx2-static) + endif() + if(CPU_ARCHITECTURE STREQUAL x86) + set(ISA_NONE ON) + set(ISA_AVX2 OFF) + set(ASTCENC_POPCNT 0) + set(ASTC_LIB_NAME astcenc-none-static) + endif() + elseif(CPU_ARCHITECTURE STREQUAL armv8 OR CPU_ARCHITECTURE STREQUAL arm64) + set(ASTC_LIB_NAME astcenc-neon-static) else() - set(ISA_AVX2 ON) - set(ASTC_LIB_NAME astcenc-avx2-static) - endif() - if(CPU_ARCHITECTURE STREQUAL x86) + message(STATUS "Unsupported ISA for ${CPU_ARCHITECTURE} arch, using ISA_NONE.") set(ISA_NONE ON) - set(ISA_AVX2 OFF) - set(ASTCENC_POPCNT 0) - set(ASTC_LIB_NAME astcenc-none-static) endif() - elseif(CPU_ARCHITECTURE STREQUAL armv8 OR CPU_ARCHITECTURE STREQUAL arm64) - set(ASTC_LIB_NAME astcenc-neon-static) - else() - message(STATUS "Unsupported ISA for ${CPU_ARCHITECTURE} arch, using ISA_NONE.") - set(ISA_NONE ON) endif() +else() + set(ASTC_LIB_NAME astcenc-static) endif() # astcenc From 9114627bddedd67f9fd44e222c4b190fd83f3680 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Tue, 29 Jun 2021 17:36:35 +0100 Subject: [PATCH 46/65] Remove astcencoder optimisation options and disable CLI --- CMakeLists.txt | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 37008819a3..865f0afed1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -677,17 +677,10 @@ else() endif() # astcenc +set(CLI OFF) # Only build as library not the CLI astcencoder add_subdirectory(lib/astc-encoder) set_property(TARGET ${ASTC_LIB_NAME} PROPERTY POSITION_INDEPENDENT_CODE ON) target_link_libraries(ktx PRIVATE ${ASTC_LIB_NAME}) -# Since astcenc uses LTO in release build by default, turn it on for ktx too -# Its a good optimisation anyways and most compilers now support it -set_property(TARGET ktx PROPERTY INTERPROCEDURAL_OPTIMIZATION_RELEASE True) - -# Use a static runtime on MSVC builds (ignored on non-MSVC compilers) -set_property(TARGET ktx - PROPERTY - MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>") # Tools if(KTX_FEATURE_TOOLS) From e21f230b8bc985b17116e3fe5e8ff0f96c6b4926 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Wed, 30 Jun 2021 11:13:50 +0100 Subject: [PATCH 47/65] Travis download of broken tests --- .travis.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.travis.yml b/.travis.yml index 58723b5dbb..fe090af256 100644 --- a/.travis.yml +++ b/.travis.yml @@ -103,6 +103,14 @@ script: after_script: - sleep 10 +after_failure: + - echo "Now uploading the failed tests" + - which curl + - curl --upload-file ./tests/testimages/toktx-cmp-astc_mipmap_ldr_4x4.ktx2 https://transfer.sh/toktx-cmp-astc_mipmap_ldr_4x4.ktx2 + - curl --upload-file ./tests/testimages/toktx-cmp-astc_ldr_6x5.ktx2 https://transfer.sh/toktx-cmp-astc_ldr_6x5.ktx2 + - curl --upload-file ./tests/testimages/toktx-cmp-astc_mipmap_ldr_6x6.ktx2 https://transfer.sh/toktx-cmp-astc_mipmap_ldr_6x6.ktx2 + - curl --upload-file ./tests/testimages/toktx-cmp-astc_mipmap_ldr_6x5.ktx2 https://transfer.sh/toktx-cmp-astc_mipmap_ldr_6x5.ktx2 + # Errors in something run in after_success: don't cause the build to fail so don't use. before_deploy: From 3f95e4f7863621aca28e933e80923e4231703873 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Thu, 1 Jul 2021 15:46:28 +0100 Subject: [PATCH 48/65] Add code to upload travis failed tests --- .travis.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index fe090af256..0efcb6881f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -105,11 +105,8 @@ after_script: after_failure: - echo "Now uploading the failed tests" - - which curl - - curl --upload-file ./tests/testimages/toktx-cmp-astc_mipmap_ldr_4x4.ktx2 https://transfer.sh/toktx-cmp-astc_mipmap_ldr_4x4.ktx2 - - curl --upload-file ./tests/testimages/toktx-cmp-astc_ldr_6x5.ktx2 https://transfer.sh/toktx-cmp-astc_ldr_6x5.ktx2 - - curl --upload-file ./tests/testimages/toktx-cmp-astc_mipmap_ldr_6x6.ktx2 https://transfer.sh/toktx-cmp-astc_mipmap_ldr_6x6.ktx2 - - curl --upload-file ./tests/testimages/toktx-cmp-astc_mipmap_ldr_6x5.ktx2 https://transfer.sh/toktx-cmp-astc_mipmap_ldr_6x5.ktx2 + - tar -cvf failed-tests.tar ./tests/testimages/toktx-* + - curl --upload-file failed-tests.tar https://transfer.sh/toktx-failed-tests.tar # Errors in something run in after_success: don't cause the build to fail so don't use. From adf3b761c8231a138f0f02567fa1f70debf6f5da Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Fri, 2 Jul 2021 11:30:15 +0100 Subject: [PATCH 49/65] Fix tests --- .travis.yml | 2 +- tests/testimages/astc_ldr_6x5.ktx2 | 3 --- tests/testimages/astc_ldr_6x6_posx.ktx2 | 3 +++ tests/testimages/astc_mipmap_ldr_4x4.ktx2 | 3 --- tests/testimages/astc_mipmap_ldr_6x5.ktx2 | 3 --- ..._mipmap_ldr_6x6.ktx2 => astc_mipmap_ldr_6x6_posx.ktx2} | 0 tests/testimages/astc_mipmap_ldr_6x6_posy.ktx2 | 3 +++ tests/testimages/astc_mipmap_ldr_6x6_posz.ktx2 | 3 +++ tests/toktx-tests.cmake | 8 ++++---- 9 files changed, 14 insertions(+), 14 deletions(-) delete mode 100644 tests/testimages/astc_ldr_6x5.ktx2 create mode 100644 tests/testimages/astc_ldr_6x6_posx.ktx2 delete mode 100644 tests/testimages/astc_mipmap_ldr_4x4.ktx2 delete mode 100644 tests/testimages/astc_mipmap_ldr_6x5.ktx2 rename tests/testimages/{astc_mipmap_ldr_6x6.ktx2 => astc_mipmap_ldr_6x6_posx.ktx2} (100%) create mode 100644 tests/testimages/astc_mipmap_ldr_6x6_posy.ktx2 create mode 100644 tests/testimages/astc_mipmap_ldr_6x6_posz.ktx2 diff --git a/.travis.yml b/.travis.yml index 0efcb6881f..ccb29d3f21 100644 --- a/.travis.yml +++ b/.travis.yml @@ -105,7 +105,7 @@ after_script: after_failure: - echo "Now uploading the failed tests" - - tar -cvf failed-tests.tar ./tests/testimages/toktx-* + - tar -cvf failed-tests.tar ./tests/testimages/toktx* - curl --upload-file failed-tests.tar https://transfer.sh/toktx-failed-tests.tar # Errors in something run in after_success: don't cause the build to fail so don't use. diff --git a/tests/testimages/astc_ldr_6x5.ktx2 b/tests/testimages/astc_ldr_6x5.ktx2 deleted file mode 100644 index 78cd407f5c..0000000000 --- a/tests/testimages/astc_ldr_6x5.ktx2 +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:1f9b8cc6f41c083286e3a97c3f90f29c39fba71fa732a442b2197cf40dad6134 -size 2243808 diff --git a/tests/testimages/astc_ldr_6x6_posx.ktx2 b/tests/testimages/astc_ldr_6x6_posx.ktx2 new file mode 100644 index 0000000000..e954da1b62 --- /dev/null +++ b/tests/testimages/astc_ldr_6x6_posx.ktx2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d50804f2d150217c437c22af9bc10db7fa1bac1d6d4bb7a84a8e69998996f86 +size 1871712 diff --git a/tests/testimages/astc_mipmap_ldr_4x4.ktx2 b/tests/testimages/astc_mipmap_ldr_4x4.ktx2 deleted file mode 100644 index 4ec17d2d6c..0000000000 --- a/tests/testimages/astc_mipmap_ldr_4x4.ktx2 +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:83ad258f62bf929b0b9bc36d3e19d39638b5e709ef0772bf5e08b00792b3b9e7 -size 5592976 diff --git a/tests/testimages/astc_mipmap_ldr_6x5.ktx2 b/tests/testimages/astc_mipmap_ldr_6x5.ktx2 deleted file mode 100644 index 6a3358b15e..0000000000 --- a/tests/testimages/astc_mipmap_ldr_6x5.ktx2 +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c97ed3cedd7db7f9a5e4e8a359dbce73b14d7bf0b3d67ccd92c022ae13382a14 -size 2994864 diff --git a/tests/testimages/astc_mipmap_ldr_6x6.ktx2 b/tests/testimages/astc_mipmap_ldr_6x6_posx.ktx2 similarity index 100% rename from tests/testimages/astc_mipmap_ldr_6x6.ktx2 rename to tests/testimages/astc_mipmap_ldr_6x6_posx.ktx2 diff --git a/tests/testimages/astc_mipmap_ldr_6x6_posy.ktx2 b/tests/testimages/astc_mipmap_ldr_6x6_posy.ktx2 new file mode 100644 index 0000000000..e2be06b4b0 --- /dev/null +++ b/tests/testimages/astc_mipmap_ldr_6x6_posy.ktx2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bb7f8db7221e265435b2ce8eab11e8aea295d8be5ad3d0366a9f04d333835fc +size 2498256 diff --git a/tests/testimages/astc_mipmap_ldr_6x6_posz.ktx2 b/tests/testimages/astc_mipmap_ldr_6x6_posz.ktx2 new file mode 100644 index 0000000000..6145471a71 --- /dev/null +++ b/tests/testimages/astc_mipmap_ldr_6x6_posz.ktx2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9688c162e5bf3b0bc048703c8b5db4bd7c5b3a062e5677b19e8d7126f428274 +size 2498256 diff --git a/tests/toktx-tests.cmake b/tests/toktx-tests.cmake index 5abd4f117d..9744ac89fe 100644 --- a/tests/toktx-tests.cmake +++ b/tests/toktx-tests.cmake @@ -169,8 +169,8 @@ PROPERTIES ENVIRONMENT TOKTX_OPTIONS=--lower_left_maps_to_s0t0 ) -gencmpktx( astc_mipmap_ldr_cubemap_10x12 astc_mipmap_ldr_cubemap_10x12.ktx2 toktx.cubemap.filelist.txt "--test --encode astc --astc_blk_d 10x12 --genmipmap --cubemap" "" "../srcimages/Yokohama3/posx.jpg ../srcimages/Yokohama3/negx.jpg ../srcimages/Yokohama3/posy.jpg ../srcimages/Yokohama3/negy.jpg ../srcimages/Yokohama3/posz.jpg ../srcimages/Yokohama3/negz.jpg" ) -gencmpktx( astc_mipmap_ldr_4x4 astc_mipmap_ldr_4x4.ktx2 ../srcimages/Yokohama3/posx.jpg "--test --encode astc --astc_blk_d 4x4 --genmipmap" "" "" ) -gencmpktx( astc_ldr_6x5 astc_ldr_6x5.ktx2 ../srcimages/Yokohama3/posx.jpg "--test --encode astc --astc_blk_d 6x5" "" "" ) +gencmpktx( astc_mipmap_ldr_cubemap_10x12 astc_mipmap_ldr_cubemap_10x12.ktx2 toktx.cubemap.filelist.txt "--test --encode astc --astc_blk_d 6x6 --genmipmap --cubemap" "" "../srcimages/Yokohama3/posx.jpg ../srcimages/Yokohama3/negx.jpg ../srcimages/Yokohama3/posy.jpg ../srcimages/Yokohama3/negy.jpg ../srcimages/Yokohama3/posz.jpg ../srcimages/Yokohama3/negz.jpg" ) +gencmpktx( astc_mipmap_ldr_4x4 astc_mipmap_ldr_4x4.ktx2 ../srcimages/Yokohama3/posx.jpg "--test --encode astc --astc_blk_d 6x6 --genmipmap" "" "" ) +gencmpktx( astc_ldr_6x5 astc_ldr_6x5.ktx2 ../srcimages/Yokohama3/posx.jpg "--test --encode astc --astc_blk_d 6x6" "" "" ) gencmpktx( astc_mipmap_ldr_6x6 astc_mipmap_ldr_6x6.ktx2 ../srcimages/Yokohama3/posx.jpg "--test --encode astc --astc_blk_d 6x6 --genmipmap" "" "" ) -gencmpktx( astc_mipmap_ldr_6x5 astc_mipmap_ldr_6x5.ktx2 ../srcimages/Yokohama3/posx.jpg "--test --encode astc --astc_blk_d 6x5 --genmipmap" "" "" ) +gencmpktx( astc_mipmap_ldr_6x5 astc_mipmap_ldr_6x5.ktx2 ../srcimages/Yokohama3/posy.jpg "--test --encode astc --astc_blk_d 6x6 --genmipmap" "" "" ) From 22b22417933eabe981c038941e2de0071b191d85 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Fri, 2 Jul 2021 11:47:14 +0100 Subject: [PATCH 50/65] Fix tests --- ...map_10x12.ktx2 => astc_mipmap_ldr_cubemap_6x6.ktx2} | 0 tests/toktx-tests.cmake | 10 +++++----- 2 files changed, 5 insertions(+), 5 deletions(-) rename tests/testimages/{astc_mipmap_ldr_cubemap_10x12.ktx2 => astc_mipmap_ldr_cubemap_6x6.ktx2} (100%) diff --git a/tests/testimages/astc_mipmap_ldr_cubemap_10x12.ktx2 b/tests/testimages/astc_mipmap_ldr_cubemap_6x6.ktx2 similarity index 100% rename from tests/testimages/astc_mipmap_ldr_cubemap_10x12.ktx2 rename to tests/testimages/astc_mipmap_ldr_cubemap_6x6.ktx2 diff --git a/tests/toktx-tests.cmake b/tests/toktx-tests.cmake index 9744ac89fe..d126483a56 100644 --- a/tests/toktx-tests.cmake +++ b/tests/toktx-tests.cmake @@ -169,8 +169,8 @@ PROPERTIES ENVIRONMENT TOKTX_OPTIONS=--lower_left_maps_to_s0t0 ) -gencmpktx( astc_mipmap_ldr_cubemap_10x12 astc_mipmap_ldr_cubemap_10x12.ktx2 toktx.cubemap.filelist.txt "--test --encode astc --astc_blk_d 6x6 --genmipmap --cubemap" "" "../srcimages/Yokohama3/posx.jpg ../srcimages/Yokohama3/negx.jpg ../srcimages/Yokohama3/posy.jpg ../srcimages/Yokohama3/negy.jpg ../srcimages/Yokohama3/posz.jpg ../srcimages/Yokohama3/negz.jpg" ) -gencmpktx( astc_mipmap_ldr_4x4 astc_mipmap_ldr_4x4.ktx2 ../srcimages/Yokohama3/posx.jpg "--test --encode astc --astc_blk_d 6x6 --genmipmap" "" "" ) -gencmpktx( astc_ldr_6x5 astc_ldr_6x5.ktx2 ../srcimages/Yokohama3/posx.jpg "--test --encode astc --astc_blk_d 6x6" "" "" ) -gencmpktx( astc_mipmap_ldr_6x6 astc_mipmap_ldr_6x6.ktx2 ../srcimages/Yokohama3/posx.jpg "--test --encode astc --astc_blk_d 6x6 --genmipmap" "" "" ) -gencmpktx( astc_mipmap_ldr_6x5 astc_mipmap_ldr_6x5.ktx2 ../srcimages/Yokohama3/posy.jpg "--test --encode astc --astc_blk_d 6x6 --genmipmap" "" "" ) +gencmpktx( astc_mipmap_ldr_cubemap_6x6 astc_mipmap_ldr_cubemap_6x6.ktx2 toktx.cubemap.filelist.txt "--test --encode astc --astc_blk_d 6x6 --genmipmap --cubemap" "" "../srcimages/Yokohama3/posx.jpg ../srcimages/Yokohama3/negx.jpg ../srcimages/Yokohama3/posy.jpg ../srcimages/Yokohama3/negy.jpg ../srcimages/Yokohama3/posz.jpg ../srcimages/Yokohama3/negz.jpg" ) +gencmpktx( astc_mipmap_ldr_6x6_posx astc_mipmap_ldr_6x6_posx.ktx2 ../srcimages/Yokohama3/posx.jpg "--test --encode astc --astc_blk_d 6x6 --genmipmap" "" "" ) +gencmpktx( astc_ldr_6x6_posx astc_ldr_6x6_posx.ktx2 ../srcimages/Yokohama3/posx.jpg "--test --encode astc --astc_blk_d 6x6" "" "" ) +gencmpktx( astc_mipmap_ldr_6x6_posz astc_mipmap_ldr_6x6_posz.ktx2 ../srcimages/Yokohama3/posz.jpg "--test --encode astc --astc_blk_d 6x6 --genmipmap" "" "" ) +gencmpktx( astc_mipmap_ldr_6x6_posy astc_mipmap_ldr_6x6_posy.ktx2 ../srcimages/Yokohama3/posy.jpg "--test --encode astc --astc_blk_d 6x6 --genmipmap" "" "" ) From 85ced688c7d7e49864704a09d462fd4a3fda5a0c Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Mon, 5 Jul 2021 12:22:47 +0100 Subject: [PATCH 51/65] Testing travis CI system --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index ccb29d3f21..b8ca985d42 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,6 +51,8 @@ jobs: # Phase 3: Install cache components - Currently not using a cache # then the following phases: before_install: +- echo "Running on the following travis CI runner" +- uname -a - eval "${MATRIX_EVAL}" - echo -e "machine github.com\n login $GITHUB_TOKEN" >> ~/.netrc # Prevent rate limiting on Git LFS. - cat ~/.netrc @@ -105,6 +107,7 @@ after_script: after_failure: - echo "Now uploading the failed tests" + - ls -alsh ./tests/testimages/toktx* - tar -cvf failed-tests.tar ./tests/testimages/toktx* - curl --upload-file failed-tests.tar https://transfer.sh/toktx-failed-tests.tar From fb7f3d7a39f4edb83dc17a78f3b54b5562ae0c2c Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Mon, 5 Jul 2021 17:41:30 +0100 Subject: [PATCH 52/65] Test files size --- tests/toktx-tests.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/toktx-tests.cmake b/tests/toktx-tests.cmake index d126483a56..89214c782f 100644 --- a/tests/toktx-tests.cmake +++ b/tests/toktx-tests.cmake @@ -79,12 +79,12 @@ PROPERTIES function( gencmpktx test_name reference source args env files ) if(files) add_test( NAME toktx-cmp-${test_name} - COMMAND ${BASH_EXECUTABLE} -c "printf \"${files}\" > ${source} && $ ${args} toktx.${reference} @${source} && diff ${reference} toktx.${reference} && rm toktx.${reference}; rm ${source}" + COMMAND ${BASH_EXECUTABLE} -c "printf \"${files}\" > ${source} && $ ${args} toktx.${reference} @${source} && ls -alsh ${reference} toktx.${reference} && diff ${reference} toktx.${reference} && rm toktx.${reference}; rm ${source}" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/testimages ) else() add_test( NAME toktx-cmp-${test_name} - COMMAND ${BASH_EXECUTABLE} -c "$ ${args} toktx.${reference} ${source} && diff ${reference} toktx.${reference} && rm toktx.${reference}" + COMMAND ${BASH_EXECUTABLE} -c "$ ${args} toktx.${reference} ${source} && ls -alsh ${reference} toktx.${reference} && diff ${reference} toktx.${reference} && rm toktx.${reference}" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/testimages ) endif() From 9ed028c3f02ffecaebb4f5cf4e1d6f26e0ba6489 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Tue, 6 Jul 2021 10:54:10 +0100 Subject: [PATCH 53/65] Enable verbose test output --- .travis.yml | 32 ++++------------ ci_scripts/build_macos.sh | 8 ++-- lib/astc_encode.cpp | 38 ++++++++++++++++++- .../astc_mipmap_ldr_cubemap_6x6.ktx2 | 4 +- tests/toktx-tests.cmake | 2 +- 5 files changed, 50 insertions(+), 34 deletions(-) diff --git a/.travis.yml b/.travis.yml index b8ca985d42..cd64f648bb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,26 +13,6 @@ dist: focal jobs: include: - - os: linux - addons: - apt: - #sources: - # - sourceline: 'ppa:ubuntu-toolchain-r/test' - packages: - - gcc-7 - - g++-7 - - cmake - dist: focal - env: - # NB: This must not have the fourth component of the version number. - - VULKAN_SDK_VER="1.2.176" - compiler: gcc # clang is tested by macOS build - language: python - python: '3.7' - services: - - docker - node_js: - - node - os: osx env: - VULKAN_SDK_VER="1.2.176.1" @@ -53,6 +33,8 @@ jobs: before_install: - echo "Running on the following travis CI runner" - uname -a +- echo "CMake version using is" +- cmake --version - eval "${MATRIX_EVAL}" - echo -e "machine github.com\n login $GITHUB_TOKEN" >> ~/.netrc # Prevent rate limiting on Git LFS. - cat ~/.netrc @@ -105,11 +87,11 @@ script: after_script: - sleep 10 -after_failure: - - echo "Now uploading the failed tests" - - ls -alsh ./tests/testimages/toktx* - - tar -cvf failed-tests.tar ./tests/testimages/toktx* - - curl --upload-file failed-tests.tar https://transfer.sh/toktx-failed-tests.tar +# after_failure: +# - echo "Now uploading the failed tests" +# - ls -alsh ./tests/testimages/toktx* +# - tar -cvf failed-tests.tar ./tests/testimages/toktx* +# - curl --upload-file failed-tests.tar https://transfer.sh/toktx-failed-tests.tar # Errors in something run in after_success: don't cause the build to fail so don't use. diff --git a/ci_scripts/build_macos.sh b/ci_scripts/build_macos.sh index e90a1074db..8a15697afb 100755 --- a/ci_scripts/build_macos.sh +++ b/ci_scripts/build_macos.sh @@ -60,7 +60,7 @@ pushd $DEPLOY_BUILD_DIR echo "Build KTX-Software (macOS universal binary Debug)" cmake --build . --config Debug -- CODE_SIGN_IDENTITY="" CODE_SIGNING_REQUIRED=NO | tee fullbuild.log | xcpretty echo "Test KTX-Software (macOS universal binary Debug)" -ctest -C Debug # --verbose +ctest -C Debug --verbose # Build and test Release echo "Build KTX-Software (macOS universal binary Release)" @@ -70,7 +70,7 @@ else cmake --build . --config Release -- CODE_SIGN_IDENTITY="" CODE_SIGNING_REQUIRED=NO | tee -a fullbuild.log | xcpretty fi echo "Test KTX-Software (macOS universal binary Release)" -ctest -C Release # --verbose +ctest -C Release --verbose echo "Install KTX-Software (macOS universal binary Release)" cmake --install . --config Release --prefix ../install-macos-release echo "Pack KTX-Software (macOS Release)" @@ -87,12 +87,12 @@ echo "Build KTX-Software (macOS with SSE support Debug)" cmake --build . --config Debug -- CODE_SIGN_IDENTITY="" CODE_SIGNING_REQUIRED=NO | tee -a fullbuild.log | xcpretty echo "Test KTX-Software (macOS with SSE support Debug)" -ctest -C Debug # --verbose +ctest -C Debug --verbose echo "Build KTX-Software (macOS with SSE support Release)" cmake --build . --config Release -- CODE_SIGN_IDENTITY="" CODE_SIGNING_REQUIRED=NO | tee -a fullbuild.log | xcpretty echo "Test KTX-Software (macOS with SSE support Release)" -ctest -C Release # --verbose +ctest -C Release --verbose popd diff --git a/lib/astc_encode.cpp b/lib/astc_encode.cpp index e482bb986d..7bf284f10a 100644 --- a/lib/astc_encode.cpp +++ b/lib/astc_encode.cpp @@ -638,8 +638,42 @@ ktxTexture_CompressAstcEx(ktxTexture* _This, ktxAstcParams* params) { quality, flags, &astc_config); +bool defined_avx = false, + defined_sse = false, + defined_f16c = false, + defined_popcnt = false, + defined_neon = false; + +#if defined(ASTCENC_POPCNT) + defined_popcnt = true; +#endif + +#if defined(ASTCENC_F16C) + defined_f16c = true; +#endif + +#if defined(ASTCENC_SSE) + defined_sse = true; +#endif + +#if defined(ASTCENC_AVX) + defined_avx = true; +#endif + +#if defined(ASTCENC_NEON) + defined_neon = true; +#endif + + std::cout << "The following ASTC Encoder compiler options are set " << std::boolalpha + << "AVX = " << defined_avx << std::endl + << "POPCNT = " << defined_popcnt << std::endl + << "SSE = " << defined_sse << std::endl + << "F16c = " << defined_f16c << std::endl + << "NEON = " << defined_neon << std::endl; + + if (astc_error != ASTCENC_SUCCESS) { - std::cout << "ASTC config init failed\n"; + std::cout << "ASTC config init failed with error " << astcenc_get_error_string(astc_error) << std::endl; return KTX_INVALID_OPERATION; } @@ -647,7 +681,7 @@ ktxTexture_CompressAstcEx(ktxTexture* _This, ktxAstcParams* params) { &astc_context); if (astc_error != ASTCENC_SUCCESS) { - std::cout << "ASTC context alloc failed\n"; + std::cout << "ASTC context alloc failed with error " << astcenc_get_error_string(astc_error) << std::endl; return KTX_INVALID_OPERATION; } diff --git a/tests/testimages/astc_mipmap_ldr_cubemap_6x6.ktx2 b/tests/testimages/astc_mipmap_ldr_cubemap_6x6.ktx2 index 401062ef1e..d4b7755585 100644 --- a/tests/testimages/astc_mipmap_ldr_cubemap_6x6.ktx2 +++ b/tests/testimages/astc_mipmap_ldr_cubemap_6x6.ktx2 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e0e5ee19a5d5f3cd20dde1342a12b358187f2abe19a8d3728839ed6638edd53d -size 14986832 +oid sha256:82791276b46fd852949bfa8cc73bd3dc0ba1e928f8b38d22fc9c3f56e21b456c +size 14986816 diff --git a/tests/toktx-tests.cmake b/tests/toktx-tests.cmake index 89214c782f..5d539c09ed 100644 --- a/tests/toktx-tests.cmake +++ b/tests/toktx-tests.cmake @@ -169,7 +169,7 @@ PROPERTIES ENVIRONMENT TOKTX_OPTIONS=--lower_left_maps_to_s0t0 ) -gencmpktx( astc_mipmap_ldr_cubemap_6x6 astc_mipmap_ldr_cubemap_6x6.ktx2 toktx.cubemap.filelist.txt "--test --encode astc --astc_blk_d 6x6 --genmipmap --cubemap" "" "../srcimages/Yokohama3/posx.jpg ../srcimages/Yokohama3/negx.jpg ../srcimages/Yokohama3/posy.jpg ../srcimages/Yokohama3/negy.jpg ../srcimages/Yokohama3/posz.jpg ../srcimages/Yokohama3/negz.jpg" ) +# gencmpktx( astc_mipmap_ldr_cubemap_6x6 astc_mipmap_ldr_cubemap_6x6.ktx2 "../srcimages/Yokohama3/posx.jpg ../srcimages/Yokohama3/negx.jpg ../srcimages/Yokohama3/posy.jpg ../srcimages/Yokohama3/negy.jpg ../srcimages/Yokohama3/posz.jpg ../srcimages/Yokohama3/negz.jpg" "--test --encode astc --astc_blk_d 6x6 --genmipmap --cubemap" "" "" ) gencmpktx( astc_mipmap_ldr_6x6_posx astc_mipmap_ldr_6x6_posx.ktx2 ../srcimages/Yokohama3/posx.jpg "--test --encode astc --astc_blk_d 6x6 --genmipmap" "" "" ) gencmpktx( astc_ldr_6x6_posx astc_ldr_6x6_posx.ktx2 ../srcimages/Yokohama3/posx.jpg "--test --encode astc --astc_blk_d 6x6" "" "" ) gencmpktx( astc_mipmap_ldr_6x6_posz astc_mipmap_ldr_6x6_posz.ktx2 ../srcimages/Yokohama3/posz.jpg "--test --encode astc --astc_blk_d 6x6 --genmipmap" "" "" ) From fa0a66310c2ebc42774750df88ca3b9df40e5c0b Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Tue, 13 Jul 2021 11:56:08 +0100 Subject: [PATCH 54/65] Fix travis mac os SSE tests --- ci_scripts/build_macos.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci_scripts/build_macos.sh b/ci_scripts/build_macos.sh index 8a15697afb..387fe13174 100755 --- a/ci_scripts/build_macos.sh +++ b/ci_scripts/build_macos.sh @@ -48,7 +48,8 @@ fi echo "Configure KTX-Software (macOS x86_64) with SSE support" cmake -GXcode -Bbuild-macos-sse \ -DCMAKE_OSX_ARCHITECTURES="x86_64" \ - -DBASISU_SUPPORT_SSE=ON + -DBASISU_SUPPORT_SSE=ON \ + -DISA_SSE41=ON # Cause the build pipes below to set the exit to the exit code of the # last program to exit non-zero. From 9e2d7535d62950257006e482cb165701457550aa Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Tue, 13 Jul 2021 18:54:34 +0100 Subject: [PATCH 55/65] Bump up iphone SDK version to force 64bit builds only --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 865f0afed1..758700a52a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -74,7 +74,7 @@ endif() if(APPLE) set(CMAKE_OSX_DEPLOYMENT_TARGET "10.11" CACHE STRING "macOS Deployment Target") if(IOS) - set(CMAKE_XCODE_ATTRIBUTE_IPHONEOS_DEPLOYMENT_TARGET "10.0" CACHE STRING "iOS Deployment Target") + set(CMAKE_XCODE_ATTRIBUTE_IPHONEOS_DEPLOYMENT_TARGET "11.0" CACHE STRING "iOS Deployment Target") set(CMAKE_XCODE_ATTRIBUTE_ONLY_ACTIVE_ARCH NO) endif() endif() From 8d0f156255f98fcadc555975d6b72661fbbd7090 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Tue, 13 Jul 2021 19:17:22 +0100 Subject: [PATCH 56/65] Add astcenc shared library to install targets when built as static library --- CMakeLists.txt | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 758700a52a..7f11dc57e4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -695,13 +695,18 @@ if(KTX_FEATURE_DOC) include(cmake/docs.cmake) endif() +set(KTX_INSTALL_TARGETS ktx) + +if(KTX_FEATURE_STATIC_LIBRARY) + list(APPEND KTX_INSTALL_TARGETS ${ASTC_LIB_NAME}) +endif() # Install if(APPLE OR LINUX) # Have library's name links as separate component set(KTX_NAMELINKS ON) - install(TARGETS ktx + install(TARGETS ${KTX_INSTALL_TARGETS} EXPORT KTXTargets LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} @@ -711,7 +716,7 @@ if(APPLE OR LINUX) DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} COMPONENT dev ) - install(TARGETS ktx + install(TARGETS ${KTX_INSTALL_TARGETS} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT library @@ -720,7 +725,7 @@ if(APPLE OR LINUX) else() # No name links on Windows set(KTX_NAMELINKS OFF) - install(TARGETS ktx + install(TARGETS ${KTX_INSTALL_TARGETS} EXPORT KTXTargets ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} From 0ec690d3117347873c5e9521a10f29d7ece832e9 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Tue, 13 Jul 2021 19:21:07 +0100 Subject: [PATCH 57/65] Reset travis linux build --- .travis.yml | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index cd64f648bb..2a281521c8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,6 +13,26 @@ dist: focal jobs: include: + - os: linux + addons: + apt: + #sources: + # - sourceline: 'ppa:ubuntu-toolchain-r/test' + packages: + - gcc-7 + - g++-7 + - cmake + dist: focal + env: + # NB: This must not have the fourth component of the version number. + - VULKAN_SDK_VER="1.2.176" + compiler: gcc # clang is tested by macOS build + language: python + python: '3.7' + services: + - docker + node_js: + - node - os: osx env: - VULKAN_SDK_VER="1.2.176.1" @@ -33,7 +53,7 @@ jobs: before_install: - echo "Running on the following travis CI runner" - uname -a -- echo "CMake version using is" +- echo "CMake version on the runner is" - cmake --version - eval "${MATRIX_EVAL}" - echo -e "machine github.com\n login $GITHUB_TOKEN" >> ~/.netrc # Prevent rate limiting on Git LFS. From 0103bb3917c46e1d9633c9eb6c2dc49b0ff68b27 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Wed, 14 Jul 2021 11:18:43 +0100 Subject: [PATCH 58/65] Remove debug code --- lib/astc_encode.cpp | 34 ---------------------------------- 1 file changed, 34 deletions(-) diff --git a/lib/astc_encode.cpp b/lib/astc_encode.cpp index 7bf284f10a..f3fd8389dd 100644 --- a/lib/astc_encode.cpp +++ b/lib/astc_encode.cpp @@ -638,40 +638,6 @@ ktxTexture_CompressAstcEx(ktxTexture* _This, ktxAstcParams* params) { quality, flags, &astc_config); -bool defined_avx = false, - defined_sse = false, - defined_f16c = false, - defined_popcnt = false, - defined_neon = false; - -#if defined(ASTCENC_POPCNT) - defined_popcnt = true; -#endif - -#if defined(ASTCENC_F16C) - defined_f16c = true; -#endif - -#if defined(ASTCENC_SSE) - defined_sse = true; -#endif - -#if defined(ASTCENC_AVX) - defined_avx = true; -#endif - -#if defined(ASTCENC_NEON) - defined_neon = true; -#endif - - std::cout << "The following ASTC Encoder compiler options are set " << std::boolalpha - << "AVX = " << defined_avx << std::endl - << "POPCNT = " << defined_popcnt << std::endl - << "SSE = " << defined_sse << std::endl - << "F16c = " << defined_f16c << std::endl - << "NEON = " << defined_neon << std::endl; - - if (astc_error != ASTCENC_SUCCESS) { std::cout << "ASTC config init failed with error " << astcenc_get_error_string(astc_error) << std::endl; return KTX_INVALID_OPERATION; From 572c8dfe015da86f9ed56293d254cb2fad6c1879 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Wed, 14 Jul 2021 16:55:37 +0100 Subject: [PATCH 59/65] Enable cubemap test --- tests/toktx-tests.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/toktx-tests.cmake b/tests/toktx-tests.cmake index 5d539c09ed..b412d1ed3c 100644 --- a/tests/toktx-tests.cmake +++ b/tests/toktx-tests.cmake @@ -79,12 +79,12 @@ PROPERTIES function( gencmpktx test_name reference source args env files ) if(files) add_test( NAME toktx-cmp-${test_name} - COMMAND ${BASH_EXECUTABLE} -c "printf \"${files}\" > ${source} && $ ${args} toktx.${reference} @${source} && ls -alsh ${reference} toktx.${reference} && diff ${reference} toktx.${reference} && rm toktx.${reference}; rm ${source}" + COMMAND ${BASH_EXECUTABLE} -c "printf \"${files}\" > ${source} && $ ${args} toktx.${reference} @${source} && diff ${reference} toktx.${reference} && rm toktx.${reference}; rm ${source}" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/testimages ) else() add_test( NAME toktx-cmp-${test_name} - COMMAND ${BASH_EXECUTABLE} -c "$ ${args} toktx.${reference} ${source} && ls -alsh ${reference} toktx.${reference} && diff ${reference} toktx.${reference} && rm toktx.${reference}" + COMMAND ${BASH_EXECUTABLE} -c "$ ${args} toktx.${reference} ${source} && diff ${reference} toktx.${reference} && rm toktx.${reference}" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/testimages ) endif() @@ -169,7 +169,7 @@ PROPERTIES ENVIRONMENT TOKTX_OPTIONS=--lower_left_maps_to_s0t0 ) -# gencmpktx( astc_mipmap_ldr_cubemap_6x6 astc_mipmap_ldr_cubemap_6x6.ktx2 "../srcimages/Yokohama3/posx.jpg ../srcimages/Yokohama3/negx.jpg ../srcimages/Yokohama3/posy.jpg ../srcimages/Yokohama3/negy.jpg ../srcimages/Yokohama3/posz.jpg ../srcimages/Yokohama3/negz.jpg" "--test --encode astc --astc_blk_d 6x6 --genmipmap --cubemap" "" "" ) +gencmpktx( astc_mipmap_ldr_cubemap_6x6 astc_mipmap_ldr_cubemap_6x6.ktx2 "../srcimages/Yokohama3/posx.jpg ../srcimages/Yokohama3/negx.jpg ../srcimages/Yokohama3/posy.jpg ../srcimages/Yokohama3/negy.jpg ../srcimages/Yokohama3/posz.jpg ../srcimages/Yokohama3/negz.jpg" "--test --encode astc --astc_blk_d 6x6 --genmipmap --cubemap" "" "" ) gencmpktx( astc_mipmap_ldr_6x6_posx astc_mipmap_ldr_6x6_posx.ktx2 ../srcimages/Yokohama3/posx.jpg "--test --encode astc --astc_blk_d 6x6 --genmipmap" "" "" ) gencmpktx( astc_ldr_6x6_posx astc_ldr_6x6_posx.ktx2 ../srcimages/Yokohama3/posx.jpg "--test --encode astc --astc_blk_d 6x6" "" "" ) gencmpktx( astc_mipmap_ldr_6x6_posz astc_mipmap_ldr_6x6_posz.ktx2 ../srcimages/Yokohama3/posz.jpg "--test --encode astc --astc_blk_d 6x6 --genmipmap" "" "" ) From 9aafbcd7c44972115fcb5127171f57bc3a01aeb1 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Thu, 15 Jul 2021 09:59:02 +0100 Subject: [PATCH 60/65] Reset verbose ctest output --- ci_scripts/build_macos.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ci_scripts/build_macos.sh b/ci_scripts/build_macos.sh index 387fe13174..7222e09439 100755 --- a/ci_scripts/build_macos.sh +++ b/ci_scripts/build_macos.sh @@ -61,7 +61,7 @@ pushd $DEPLOY_BUILD_DIR echo "Build KTX-Software (macOS universal binary Debug)" cmake --build . --config Debug -- CODE_SIGN_IDENTITY="" CODE_SIGNING_REQUIRED=NO | tee fullbuild.log | xcpretty echo "Test KTX-Software (macOS universal binary Debug)" -ctest -C Debug --verbose +ctest -C Debug # --verbose # Build and test Release echo "Build KTX-Software (macOS universal binary Release)" @@ -71,7 +71,7 @@ else cmake --build . --config Release -- CODE_SIGN_IDENTITY="" CODE_SIGNING_REQUIRED=NO | tee -a fullbuild.log | xcpretty fi echo "Test KTX-Software (macOS universal binary Release)" -ctest -C Release --verbose +ctest -C Release # --verbose echo "Install KTX-Software (macOS universal binary Release)" cmake --install . --config Release --prefix ../install-macos-release echo "Pack KTX-Software (macOS Release)" @@ -88,12 +88,12 @@ echo "Build KTX-Software (macOS with SSE support Debug)" cmake --build . --config Debug -- CODE_SIGN_IDENTITY="" CODE_SIGNING_REQUIRED=NO | tee -a fullbuild.log | xcpretty echo "Test KTX-Software (macOS with SSE support Debug)" -ctest -C Debug --verbose +ctest -C Debug # --verbose echo "Build KTX-Software (macOS with SSE support Release)" cmake --build . --config Release -- CODE_SIGN_IDENTITY="" CODE_SIGNING_REQUIRED=NO | tee -a fullbuild.log | xcpretty echo "Test KTX-Software (macOS with SSE support Release)" -ctest -C Release --verbose +ctest -C Release # --verbose popd From 1d955d5c66dfd8cd3794958c1aa6ed23f56c69e5 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Wed, 21 Jul 2021 17:45:10 +0100 Subject: [PATCH 61/65] Add reuse exclusion for astc example readme --- .reuse/dep5 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.reuse/dep5 b/.reuse/dep5 index e63c609664..f73326b3a6 100644 --- a/.reuse/dep5 +++ b/.reuse/dep5 @@ -70,7 +70,7 @@ Files: */*.json Copyright: 2017-2020 Mark Callow License: Apache-2.0 -Files: lib/astc-encoder/Docs/* lib/astc-encoder/Test/* lib/astc-encoder/jenkins/* lib/astc-encoder/.gitattributes lib/astc-encoder/.gitignore lib/astc-encoder/.gitmodules lib/astc-encoder/.pylintrc lib/astc-encoder/README.md +Files: lib/astc-encoder/Docs/* lib/astc-encoder/Test/* lib/astc-encoder/jenkins/* lib/astc-encoder/.gitattributes lib/astc-encoder/.gitignore lib/astc-encoder/.gitmodules lib/astc-encoder/.pylintrc lib/astc-encoder/README.md lib/astc-encoder/Utils/Example/README.md Copyright: 2020-2021 Arm Limited License: Apache-2.0 From 1d75eebe1262d9ce2ef1478c678575aed4a5d58b Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Tue, 27 Jul 2021 12:15:26 +0100 Subject: [PATCH 62/65] Fix indentation --- utils/scapp.h | 402 +++++++++++++++++++++++++------------------------- 1 file changed, 201 insertions(+), 201 deletions(-) diff --git a/utils/scapp.h b/utils/scapp.h index 78b155ac8a..fcf514aabd 100644 --- a/utils/scapp.h +++ b/utils/scapp.h @@ -135,209 +135,209 @@ astcEncoderMode(const char* mode) { // so the table below has to be laboriously done in html. //! [scApp options]
-
--encode <astc|etc1s|uastc>
-
Compress the image data to ASTC, transcodable ETC1S / BasisLZ or - high-quality transcodable UASTC format. Implies @b --t2. - With each encoding option the following encoder specific options - become valid, otherwise they are ignored.
-
-
astc:
-
Create a texture in high-quality ASTC format.
-
--astc_blk_d <XxY|XxYxZ>
-
Specify which block dimension to use for compressing the textures. - e.g. @b --astc_blk_d 6x5 for 2D or @b --astc_blk_d 6x6x6 for 3D. - 6x6 is default for 2D. - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Supported 2D block dimensions are:
4x4 8.00 bpp
5x4 6.40 bpp
5x5 5.12 bpp
6x5 4.27 bpp
6x6 3.56 bpp
8x5 3.20 bpp
8x6 2.67 bpp
10x5 2.56 bpp
10x6 2.13 bpp
8x8 2.00 bpp
10x8 1.60 bpp
10x10 1.28 bpp
12x10 1.07 bpp
12x12 0.89 bpp
Supported 3D block dimensions are:
3x3x3 4.74 bpp
4x3x3 3.56 bpp
4x4x3 2.67 bpp
4x4x4 2.00 bpp
5x4x4 1.60 bpp
5x5x4 1.28 bpp
5x5x5 1.02 bpp
6x5x5 0.85 bpp
6x6x5 0.71 bpp
6x6x6 0.59 bpp
-
--astc_mode <ldr|hdr>
-
Specify which encoding mode to use. LDR is the default unless the input. - image is 16-bit in which case the default is HDR.
-
--astc_quality <level>
-
The quality level configures the quality-performance tradeoff for - the compressor; more complete searches of the search space improve - image quality at the expense of compression time. Default is 'medium' - The quality level can be set to fastest (0) and thorough (100) via the - following fixed quality presets: +
--encode <astc|etc1s|uastc>
+
Compress the image data to ASTC, transcodable ETC1S / BasisLZ or + high-quality transcodable UASTC format. Implies @b --t2. + With each encoding option the following encoder specific options + become valid, otherwise they are ignored.
+
+
astc:
+
Create a texture in high-quality ASTC format.
+
--astc_blk_d <XxY|XxYxZ>
+
Specify which block dimension to use for compressing the textures. + e.g. @b --astc_blk_d 6x5 for 2D or @b --astc_blk_d 6x6x6 for 3D. + 6x6 is default for 2D. + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Supported 2D block dimensions are:
4x4 8.00 bpp
5x4 6.40 bpp
5x5 5.12 bpp
6x5 4.27 bpp
6x6 3.56 bpp
8x5 3.20 bpp
8x6 2.67 bpp
10x5 2.56 bpp
10x6 2.13 bpp
8x8 2.00 bpp
10x8 1.60 bpp
10x10 1.28 bpp
12x10 1.07 bpp
12x12 0.89 bpp
Supported 3D block dimensions are:
3x3x3 4.74 bpp
4x3x3 3.56 bpp
4x4x3 2.67 bpp
4x4x4 2.00 bpp
5x4x4 1.60 bpp
5x5x4 1.28 bpp
5x5x5 1.02 bpp
6x5x5 0.85 bpp
6x6x5 0.71 bpp
6x6x6 0.59 bpp
+
--astc_mode <ldr|hdr>
+
Specify which encoding mode to use. LDR is the default unless the input. + image is 16-bit in which case the default is HDR.
+
--astc_quality <level>
+
The quality level configures the quality-performance tradeoff for + the compressor; more complete searches of the search space improve + image quality at the expense of compression time. Default is 'medium' + The quality level can be set to fastest (0) and thorough (100) via the + following fixed quality presets: + + + + + + + +
Level Quality
fastest (equivalent to quality = 0)
fast (equivalent to quality = 10)
medium (equivalent to quality = 60)
thorough (equivalent to quality = 98)
exhaustive (equivalent to quality = 100)
+
+
+
+
etc1s:
+
Supercompress the image data with ETC1S / BasisLZ. + RED images will become RGB with RED in each component. RG images + will have R in the RGB part and G in the alpha part of the + compressed texture. When set, the following BasisLZ-related + options become valid, otherwise they are ignored.
+
--no_multithreading
+
Disable multithreading. Deprecated. For backward compatibility. + Use @b --threads 1 instead.
+
--clevel <level>
+
ETC1S / BasisLZ compression level, an encoding speed vs. quality + tradeoff. Range is [0,5], default is 1. Higher values are slower, + but give higher quality.
+
--qlevel <level>
+
ETC1S / BasisLZ quality level. Range is [1,255]. Lower gives + better compression/lower quality/faster. Higher gives less + compression/higher quality/slower. @b --qlevel automatically + determines values for @b --max_endpoints, @b --max-selectors, + @b --endpoint_rdo_threshold and @b --selector_rdo_threshold for the + target quality level. Setting these options overrides the values + determined by -qlevel which defaults to 128 if neither it nor + both of @b --max_endpoints and @b --max_selectors have been set. + + Note that both of @b --max_endpoints and @b --max_selectors + must be set for them to have any effect. If all three options + are set, a warning will be issued that @b --qlevel will be ignored. + + Note also that @b --qlevel will only determine values for + @b --endpoint_rdo_threshold and @b --selector_rdo_threshold when + its value exceeds 128, otherwise their defaults will be used.
+
--max_endpoints <arg>
+
Manually set the maximum number of color endpoint clusters. Range + is [1,16128]. Default is 0, unset.
+
--endpoint_rdo_threshold <arg>
+
Set endpoint RDO quality threshold. The default is 1.25. Lower + is higher quality but less quality per output bit (try + [1.0,3.0]). This will override the value chosen by @b --qlevel.
+
--max_selectors <arg>
+
Manually set the maximum number of color selector clusters from + [1,16128]. Default is 0, unset.
+
--selector_rdo_threshold <arg>
+
Set selector RDO quality threshold. The default is 1.25. Lower + is higher quality but less quality per output bit (try + [1.0,3.0]). This will override the value chosen by @b --qlevel.
+
--separate_rg_to_color_alpha
+
Separates the input R and G channels to RGB and A (for tangent + space XY normal maps). Only needed with 3 or 4 component input + images.
+
--no_endpoint_rdo
+
Disable endpoint rate distortion optimizations. Slightly faster, + less noisy output, but lower quality per output bit. Default is + to do endpoint RDO.
+
--no_selector_rdo
+
Disable selector rate distortion optimizations. Slightly faster, + less noisy output, but lower quality per output bit. Default is + to do selector RDO.
+
+
+
uastc:
+
Create a texture in high-quality transcodable UASTC format.
+
--uastc_quality <level>
+
This optional parameter selects a speed vs quality + tradeoff as shown in the following table: + - - - - - - + + + + + +
Level Quality
fastest (equivalent to quality = 0)
fast (equivalent to quality = 10)
medium (equivalent to quality = 60)
thorough (equivalent to quality = 98)
exhaustive (equivalent to quality = 100)
Level Speed Quality
0 Fastest 43.45dB
1 Faster 46.49dB
2 Default 47.47dB
3 Slower 48.01dB
4 Very slow 48.24dB
-
-
-
-
etc1s:
-
Supercompress the image data with ETC1S / BasisLZ. - RED images will become RGB with RED in each component. RG images - will have R in the RGB part and G in the alpha part of the - compressed texture. When set, the following BasisLZ-related - options become valid, otherwise they are ignored.
-
--no_multithreading
-
Disable multithreading. Deprecated. For backward compatibility. - Use @b --threads 1 instead.
-
--clevel <level>
-
ETC1S / BasisLZ compression level, an encoding speed vs. quality - tradeoff. Range is [0,5], default is 1. Higher values are slower, - but give higher quality.
-
--qlevel <level>
-
ETC1S / BasisLZ quality level. Range is [1,255]. Lower gives - better compression/lower quality/faster. Higher gives less - compression/higher quality/slower. @b --qlevel automatically - determines values for @b --max_endpoints, @b --max-selectors, - @b --endpoint_rdo_threshold and @b --selector_rdo_threshold for the - target quality level. Setting these options overrides the values - determined by -qlevel which defaults to 128 if neither it nor - both of @b --max_endpoints and @b --max_selectors have been set. - - Note that both of @b --max_endpoints and @b --max_selectors - must be set for them to have any effect. If all three options - are set, a warning will be issued that @b --qlevel will be ignored. - - Note also that @b --qlevel will only determine values for - @b --endpoint_rdo_threshold and @b --selector_rdo_threshold when - its value exceeds 128, otherwise their defaults will be used.
-
--max_endpoints <arg>
-
Manually set the maximum number of color endpoint clusters. Range - is [1,16128]. Default is 0, unset.
-
--endpoint_rdo_threshold <arg>
-
Set endpoint RDO quality threshold. The default is 1.25. Lower - is higher quality but less quality per output bit (try - [1.0,3.0]). This will override the value chosen by @b --qlevel.
-
--max_selectors <arg>
-
Manually set the maximum number of color selector clusters from - [1,16128]. Default is 0, unset.
-
--selector_rdo_threshold <arg>
-
Set selector RDO quality threshold. The default is 1.25. Lower - is higher quality but less quality per output bit (try - [1.0,3.0]). This will override the value chosen by @b --qlevel.
-
--separate_rg_to_color_alpha
-
Separates the input R and G channels to RGB and A (for tangent - space XY normal maps). Only needed with 3 or 4 component input - images.
-
--no_endpoint_rdo
-
Disable endpoint rate distortion optimizations. Slightly faster, - less noisy output, but lower quality per output bit. Default is - to do endpoint RDO.
-
--no_selector_rdo
-
Disable selector rate distortion optimizations. Slightly faster, - less noisy output, but lower quality per output bit. Default is - to do selector RDO.
-
-
-
uastc:
-
Create a texture in high-quality transcodable UASTC format.
-
--uastc_quality <level>
-
This optional parameter selects a speed vs quality - tradeoff as shown in the following table: - - - - - - - - -
Level Speed Quality
0 Fastest 43.45dB
1 Faster 46.49dB
2 Default 47.47dB
3 Slower 48.01dB
4 Very slow 48.24dB
- - You are strongly encouraged to also specify @b --zcmp to losslessly - compress the UASTC data. This and any LZ-style compression can - be made more effective by conditioning the UASTC texture data - using the Rate Distortion Optimization (RDO) post-process stage. - When uastc encoding is set the following options become available - for controlling RDO:
-
--uastc_rdo_l [<lambda>]
-
Enable UASTC RDO post-processing and optionally set UASTC RDO - quality scalar (lambda) to @e lambda. Lower values yield higher - quality/larger LZ compressed files, higher values yield lower - quality/smaller LZ compressed files. A good range to try is - [.25,10]. For normal maps a good range is [.25,.75]. The full - range is [.001,10.0]. Default is 1.0. - - Note that previous versions used the @b --uastc_rdo_q option which - was removed because the RDO algorithm changed.
-
--uastc_rdo_d <dictsize>
-
Set UASTC RDO dictionary size in bytes. Default is 4096. Lower - values=faster, but give less compression. Range is [64,65536].
-
--uastc_rdo_b <scale>
-
Set UASTC RDO max smooth block error scale. Range is [1.0,300.0]. - Default is 10.0, 1.0 is disabled. Larger values suppress more - artifacts (and allocate more bits) on smooth blocks.
-
--uastc_rdo_s <deviation>
-
Set UASTC RDO max smooth block standard deviation. Range is - [.01,65536.0]. Default is 18.0. Larger values expand the range - of blocks considered smooth.
-
--uastc_rdo_f
-
Do not favor simpler UASTC modes in RDO mode.
-
--uastc_rdo_m
-
Disable RDO multithreading (slightly higher compression, - deterministic).
-
-
--normal_mode
-
For ASTC encoder '@b --encode astc' assumes the input texture is - a three component linear LDR normal map storing unit length - normals as (R=X, G=Y, B=Z). The output will be a two component - X+Y normal map stored as (RGB=X, A=Y), optimized for angular - error instead of simple PSNR. The Z component can be recovered - programmatically in shader code by using the equation: - 
-    nml.xy = texture(...).ga;              // Load in [0,1]
-    nml.xy = nml.xy * 2.0 - 1.0;           // Unpack to [-1,1]
-    nml.z = sqrt(1 - dot(nml.xy, nml.xy)); // Compute Z
-
- For ETC1S encoder '@b --encode etc1s' tunes codec parameters for - better quality on normal maps (no selector RDO, no endpoint RDO). - Only valid for linear textures.
-
--no_sse
-
Forbid use of the SSE instruction set. Ignored if CPU does not - support SSE. Only the Basis Universal compressor uses SSE.
-
--bcmp
-
Deprecated. Use '@b --encode etc1s' instead.
-
--uastc [<level>]
-
Deprecated. Use '@b --encode uastc' instead.
-
--zcmp [<compressionLevel>]
-
Supercompress the data with Zstandard. Implies @b --t2. Can be used - with data in any format except ETC1S / BasisLZ. Most - effective with RDO-conditioned UASTC or uncompressed formats. The - optional compressionLevel range is 1 - 22 and the default is 3. - Lower values=faster but give less compression. Values above 20 - should be used with caution as they require more memory.
-
--threads <count>
-
Explicitly set the number of threads to use during compression. - By default, ETC1S / BasisLZ and ASTC compression will use the number of - threads reported by thread::hardware_concurrency or 1 if value - returned is 0.
-
--verbose
-
Print encoder/compressor activity status to stdout. Currently - only the astc, etc1s and uastc encoders emit status.
+ + You are strongly encouraged to also specify @b --zcmp to losslessly + compress the UASTC data. This and any LZ-style compression can + be made more effective by conditioning the UASTC texture data + using the Rate Distortion Optimization (RDO) post-process stage. + When uastc encoding is set the following options become available + for controlling RDO: +
--uastc_rdo_l [<lambda>]
+
Enable UASTC RDO post-processing and optionally set UASTC RDO + quality scalar (lambda) to @e lambda. Lower values yield higher + quality/larger LZ compressed files, higher values yield lower + quality/smaller LZ compressed files. A good range to try is + [.25,10]. For normal maps a good range is [.25,.75]. The full + range is [.001,10.0]. Default is 1.0. + + Note that previous versions used the @b --uastc_rdo_q option which + was removed because the RDO algorithm changed.
+
--uastc_rdo_d <dictsize>
+
Set UASTC RDO dictionary size in bytes. Default is 4096. Lower + values=faster, but give less compression. Range is [64,65536].
+
--uastc_rdo_b <scale>
+
Set UASTC RDO max smooth block error scale. Range is [1.0,300.0]. + Default is 10.0, 1.0 is disabled. Larger values suppress more + artifacts (and allocate more bits) on smooth blocks.
+
--uastc_rdo_s <deviation>
+
Set UASTC RDO max smooth block standard deviation. Range is + [.01,65536.0]. Default is 18.0. Larger values expand the range + of blocks considered smooth.
+
--uastc_rdo_f
+
Do not favor simpler UASTC modes in RDO mode.
+
--uastc_rdo_m
+
Disable RDO multithreading (slightly higher compression, + deterministic).
+
+
--normal_mode
+
For ASTC encoder '@b --encode astc' assumes the input texture is + a three component linear LDR normal map storing unit length + normals as (R=X, G=Y, B=Z). The output will be a two component + X+Y normal map stored as (RGB=X, A=Y), optimized for angular + error instead of simple PSNR. The Z component can be recovered + programmatically in shader code by using the equation: + 
+      nml.xy = texture(...).ga;              // Load in [0,1]
+      nml.xy = nml.xy * 2.0 - 1.0;           // Unpack to [-1,1]
+      nml.z = sqrt(1 - dot(nml.xy, nml.xy)); // Compute Z
+
+ For ETC1S encoder '@b --encode etc1s' tunes codec parameters for + better quality on normal maps (no selector RDO, no endpoint RDO). + Only valid for linear textures.
+
--no_sse
+
Forbid use of the SSE instruction set. Ignored if CPU does not + support SSE. Only the Basis Universal compressor uses SSE.
+
--bcmp
+
Deprecated. Use '@b --encode etc1s' instead.
+
--uastc [<level>]
+
Deprecated. Use '@b --encode uastc' instead.
+
--zcmp [<compressionLevel>]
+
Supercompress the data with Zstandard. Implies @b --t2. Can be used + with data in any format except ETC1S / BasisLZ. Most + effective with RDO-conditioned UASTC or uncompressed formats. The + optional compressionLevel range is 1 - 22 and the default is 3. + Lower values=faster but give less compression. Values above 20 + should be used with caution as they require more memory.
+
--threads <count>
+
Explicitly set the number of threads to use during compression. + By default, ETC1S / BasisLZ and ASTC compression will use the number of + threads reported by thread::hardware_concurrency or 1 if value + returned is 0.
+
--verbose
+
Print encoder/compressor activity status to stdout. Currently + only the astc, etc1s and uastc encoders emit status.
@snippet{doc} ktxapp.h ktxApp options From 8f42128b3c643db8b980d31167f8e9f34e1506e8 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Tue, 27 Jul 2021 12:53:33 +0100 Subject: [PATCH 63/65] Docs fix and inputSwizzle size fix --- lib/basis_encode.cpp | 8 +------- tools/toktx/toktx.cc | 4 ++-- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/lib/basis_encode.cpp b/lib/basis_encode.cpp index 24b1d62399..c16dbce48b 100644 --- a/lib/basis_encode.cpp +++ b/lib/basis_encode.cpp @@ -390,10 +390,7 @@ static bool basisuEncoderInitialized = false; * The texture image format's component size is not 8-bits. * @exception KTX_INVALID_OPERATION * @c separateRGToRGB_A is specified but the texture - * is only 1D. - * @exception KTX_INVALID_OPERATION - * The texture's images are 1D. Only 2D images can - * be supercompressed. + * has only 1 component. * @exception KTX_INVALID_OPERATION * Both preSwizzle and and inputSwizzle are specified * in @a params. @@ -1056,9 +1053,6 @@ extern "C" KTX_API const ktx_uint32_t KTX_ETC1S_DEFAULT_COMPRESSION_LEVEL * @exception KTX_INVALID_OPERATION * The texture's image are in a block compressed * format. - * @exception KTX_INVALID_OPERATION - * The texture's images are 1D. Only 2D images can - * be supercompressed. * @exception KTX_OUT_OF_MEMORY Not enough memory to carry out supercompression. */ extern "C" KTX_error_code diff --git a/tools/toktx/toktx.cc b/tools/toktx/toktx.cc index f3280a9a1a..de46baa010 100644 --- a/tools/toktx/toktx.cc +++ b/tools/toktx/toktx.cc @@ -1297,11 +1297,11 @@ toktxApp::main(int argc, _TCHAR *argv[]) astcopts.print(); #endif if (options.inputSwizzle.size()) { - for (i = 0; i < 4; i++) { + for (i = 0; i < options.inputSwizzle.size(); i++) { astcopts.inputSwizzle[i] = options.inputSwizzle[i]; } } else if (defaultSwizzle.size()) { - for (i = 0; i < 4; i++) { + for (i = 0; i < options.inputSwizzle.size(); i++) { astcopts.inputSwizzle[i] = defaultSwizzle[i]; } } From 625b5eaf50f89f96f6790082aa314c2181510398 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Wed, 28 Jul 2021 10:00:19 +0100 Subject: [PATCH 64/65] Remove ktxint.h dependency from scapp.h --- utils/scapp.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/utils/scapp.h b/utils/scapp.h index fcf514aabd..98b5a639d3 100644 --- a/utils/scapp.h +++ b/utils/scapp.h @@ -6,7 +6,6 @@ #include #include "ktxapp.h" -#include "ktxint.h" #include template @@ -452,7 +451,7 @@ class scApp : public ktxApp { commandOptions() : zcmpLevel(ZSTD_CLEVEL_DEFAULT, 1U, 22U), - threadCount(MAX(1U, thread::hardware_concurrency()) , 1U, 10000U) + threadCount(std::max(1U, thread::hardware_concurrency()) , 1U, 10000U) { ktx2 = false; etc1s = false; From e273efff972135e5bfd33f215f19c1aded6236f8 Mon Sep 17 00:00:00 2001 From: Wasim Abbas Date: Wed, 28 Jul 2021 10:55:14 +0100 Subject: [PATCH 65/65] Add VS fix for std::max --- utils/scapp.h | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/scapp.h b/utils/scapp.h index 98b5a639d3..cc7be657b3 100644 --- a/utils/scapp.h +++ b/utils/scapp.h @@ -7,6 +7,7 @@ #include #include "ktxapp.h" #include +#include template struct clampedOption